Note: This is test shard 2 of 8.
[==========] Running 8 tests from 6 test suites.
[----------] Global test environment set-up.
[----------] 1 test from RebalanceIgnoredTserversTest
[ RUN ] RebalanceIgnoredTserversTest.Basic
WARNING: Logging before InitGoogleLogging() is written to STDERR
I20240502 07:58:06.022867 16726 test_util.cc:268] Using random seed: -157886267
I20240502 07:58:06.028275 16726 ts_itest-base.cc:112] Starting cluster with:
I20240502 07:58:06.028326 16726 ts_itest-base.cc:113] --------------
I20240502 07:58:06.028340 16726 ts_itest-base.cc:114] 5 tablet servers
I20240502 07:58:06.028359 16726 ts_itest-base.cc:115] 3 replicas per TS
I20240502 07:58:06.028373 16726 ts_itest-base.cc:116] --------------
2024-05-02T07:58:06Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T07:58:06Z Disabled control of system clock
I20240502 07:58:06.036310 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.16.85.190:42017
--webserver_interface=127.16.85.190
--webserver_port=0
--builtin_ntp_servers=127.16.85.148:33259
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.16.85.190:42017
--enable_range_replica_placement=false with env {}
W20240502 07:58:06.124744 16738 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:06.124912 16738 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:06.124944 16738 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:06.127431 16738 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 07:58:06.127472 16738 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:06.127488 16738 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 07:58:06.127504 16738 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 07:58:06.130811 16738 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33259
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.16.85.190:42017
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.16.85.190:42017
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.16.85.190
--webserver_port=0
--never_fsync=true
--heap_profile_path=/tmp/kudu.16738
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:06.131455 16738 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:06.132315 16738 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:06.138216 16746 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:06.138782 16743 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:06.139688 16744 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:06.140374 16738 server_base.cc:1034] running on GCE node
I20240502 07:58:06.140797 16738 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:06.141516 16738 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:06.142652 16738 hybrid_clock.cc:648] HybridClock initialized: now 1714636686142637 us; error 26 us; skew 500 ppm
I20240502 07:58:06.144083 16738 webserver.cc:460] Webserver started at http://127.16.85.190:37161/ using document root <none> and password file <none>
I20240502 07:58:06.144521 16738 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:06.144565 16738 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:06.144753 16738 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:06.146116 16738 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "a7d213ce3c8d4b5a92609f0675753754"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.146502 16738 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "a7d213ce3c8d4b5a92609f0675753754"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.149272 16738 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.001s sys 0.003s
I20240502 07:58:06.151088 16752 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:06.151873 16738 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:06.151978 16738 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
uuid: "a7d213ce3c8d4b5a92609f0675753754"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.152042 16738 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:06.166224 16738 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:06.166707 16738 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:06.166831 16738 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:06.172359 16738 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.190:42017
I20240502 07:58:06.172372 16804 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.190:42017 every 8 connection(s)
I20240502 07:58:06.173135 16738 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 07:58:06.175184 16805 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:06.179279 16805 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754: Bootstrap starting.
I20240502 07:58:06.180568 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 16738
I20240502 07:58:06.180706 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 07:58:06.181205 16805 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:06.181766 16805 log.cc:824] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:06.183236 16805 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754: No bootstrap required, opened a new log
I20240502 07:58:06.185635 16805 raft_consensus.cc:348] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a7d213ce3c8d4b5a92609f0675753754" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 42017 } }
I20240502 07:58:06.185791 16805 raft_consensus.cc:374] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:06.185858 16805 raft_consensus.cc:729] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a7d213ce3c8d4b5a92609f0675753754, State: Initialized, Role: FOLLOWER
I20240502 07:58:06.186193 16805 consensus_queue.cc:260] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a7d213ce3c8d4b5a92609f0675753754" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 42017 } }
I20240502 07:58:06.186300 16805 raft_consensus.cc:388] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:06.186362 16805 raft_consensus.cc:482] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:06.186465 16805 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:06.187228 16805 raft_consensus.cc:504] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a7d213ce3c8d4b5a92609f0675753754" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 42017 } }
I20240502 07:58:06.187511 16805 leader_election.cc:302] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: a7d213ce3c8d4b5a92609f0675753754; no voters:
I20240502 07:58:06.187729 16805 leader_election.cc:288] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:06.187976 16810 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:06.188184 16810 raft_consensus.cc:686] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [term 1 LEADER]: Becoming Leader. State: Replica: a7d213ce3c8d4b5a92609f0675753754, State: Running, Role: LEADER
I20240502 07:58:06.188457 16810 consensus_queue.cc:237] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a7d213ce3c8d4b5a92609f0675753754" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 42017 } }
I20240502 07:58:06.188580 16805 sys_catalog.cc:564] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [sys.catalog]: configured and running, proceeding with master startup.
I20240502 07:58:06.189800 16811 sys_catalog.cc:455] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "a7d213ce3c8d4b5a92609f0675753754" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a7d213ce3c8d4b5a92609f0675753754" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 42017 } } }
I20240502 07:58:06.189916 16811 sys_catalog.cc:458] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [sys.catalog]: This master's current role is: LEADER
I20240502 07:58:06.190193 16812 sys_catalog.cc:455] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a7d213ce3c8d4b5a92609f0675753754. Latest consensus state: current_term: 1 leader_uuid: "a7d213ce3c8d4b5a92609f0675753754" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a7d213ce3c8d4b5a92609f0675753754" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 42017 } } }
I20240502 07:58:06.190243 16819 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 07:58:06.190339 16812 sys_catalog.cc:458] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754 [sys.catalog]: This master's current role is: LEADER
I20240502 07:58:06.192822 16819 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 07:58:06.198884 16819 catalog_manager.cc:1347] Generated new cluster ID: d9dabcb459304cf89a8522f0af4826dc
I20240502 07:58:06.198947 16819 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 07:58:06.215996 16819 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 07:58:06.216928 16819 catalog_manager.cc:1504] Loading token signing keys...
I20240502 07:58:06.224418 16819 catalog_manager.cc:5857] T 00000000000000000000000000000000 P a7d213ce3c8d4b5a92609f0675753754: Generated new TSK 0
I20240502 07:58:06.225019 16819 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 07:58:06.238185 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.129:0
--local_ip_for_outbound_sockets=127.16.85.129
--webserver_interface=127.16.85.129
--webserver_port=0
--tserver_master_addrs=127.16.85.190:42017
--builtin_ntp_servers=127.16.85.148:33259
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_cache_size_limit_mb=10
--consensus_rpc_timeout_ms=30000 with env {}
W20240502 07:58:06.343677 16829 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:06.343853 16829 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:06.343916 16829 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:06.346495 16829 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:06.346607 16829 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.129
I20240502 07:58:06.349747 16829 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33259
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_cache_size_limit_mb=10
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.16.85.129
--webserver_port=0
--tserver_master_addrs=127.16.85.190:42017
--never_fsync=true
--heap_profile_path=/tmp/kudu.16829
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.129
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:06.350492 16829 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:06.351403 16829 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:06.358079 16837 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:06.358691 16834 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:06.359387 16835 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:06.360114 16829 server_base.cc:1034] running on GCE node
I20240502 07:58:06.360389 16829 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:06.360852 16829 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:06.361981 16829 hybrid_clock.cc:648] HybridClock initialized: now 1714636686361970 us; error 23 us; skew 500 ppm
I20240502 07:58:06.363407 16829 webserver.cc:460] Webserver started at http://127.16.85.129:45855/ using document root <none> and password file <none>
I20240502 07:58:06.363852 16829 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:06.363903 16829 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:06.364045 16829 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:06.365427 16829 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "de577719e9174df89ddd3e411320a1e8"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.365824 16829 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "de577719e9174df89ddd3e411320a1e8"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.368526 16829 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 07:58:06.370410 16843 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:06.371188 16829 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:06.371276 16829 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "de577719e9174df89ddd3e411320a1e8"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.371342 16829 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:06.383555 16829 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:06.384001 16829 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:06.384124 16829 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:06.384528 16829 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:06.385229 16829 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:06.385272 16829 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:06.385318 16829 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:06.385346 16829 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:06.392483 16829 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.129:42105
I20240502 07:58:06.392498 16956 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.129:42105 every 8 connection(s)
I20240502 07:58:06.393246 16829 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 07:58:06.393620 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 16829
I20240502 07:58:06.393712 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 07:58:06.395609 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.130:0
--local_ip_for_outbound_sockets=127.16.85.130
--webserver_interface=127.16.85.130
--webserver_port=0
--tserver_master_addrs=127.16.85.190:42017
--builtin_ntp_servers=127.16.85.148:33259
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_cache_size_limit_mb=10
--consensus_rpc_timeout_ms=30000 with env {}
I20240502 07:58:06.401114 16957 heartbeater.cc:347] Connected to a master server at 127.16.85.190:42017
I20240502 07:58:06.401326 16957 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:06.401911 16957 heartbeater.cc:511] Master 127.16.85.190:42017 requested a full tablet report, sending...
I20240502 07:58:06.403146 16769 ts_manager.cc:194] Registered new tserver with Master: de577719e9174df89ddd3e411320a1e8 (127.16.85.129:42105)
I20240502 07:58:06.404119 16769 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.129:55285
W20240502 07:58:06.487362 16960 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:06.487524 16960 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:06.487557 16960 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:06.490599 16960 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:06.490700 16960 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.130
I20240502 07:58:06.493669 16960 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33259
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_cache_size_limit_mb=10
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.16.85.130
--webserver_port=0
--tserver_master_addrs=127.16.85.190:42017
--never_fsync=true
--heap_profile_path=/tmp/kudu.16960
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.130
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:06.494336 16960 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:06.495262 16960 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:06.501935 16965 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:06.502451 16966 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:06.503041 16968 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:06.503710 16960 server_base.cc:1034] running on GCE node
I20240502 07:58:06.503983 16960 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:06.504386 16960 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:06.505517 16960 hybrid_clock.cc:648] HybridClock initialized: now 1714636686505504 us; error 28 us; skew 500 ppm
I20240502 07:58:06.506951 16960 webserver.cc:460] Webserver started at http://127.16.85.130:38123/ using document root <none> and password file <none>
I20240502 07:58:06.507393 16960 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:06.507460 16960 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:06.507622 16960 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:06.509078 16960 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "6e77183c846f4c3dbed2acb777d12f9a"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.509514 16960 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "6e77183c846f4c3dbed2acb777d12f9a"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.512243 16960 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 07:58:06.514034 16974 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:06.514886 16960 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:06.514983 16960 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "6e77183c846f4c3dbed2acb777d12f9a"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.515053 16960 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:06.533095 16960 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:06.533630 16960 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:06.533769 16960 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:06.534219 16960 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:06.534938 16960 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:06.534981 16960 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:06.535028 16960 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:06.535058 16960 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:06.542591 16960 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.130:43625
I20240502 07:58:06.542616 17087 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.130:43625 every 8 connection(s)
I20240502 07:58:06.543365 16960 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 07:58:06.550314 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 16960
I20240502 07:58:06.550377 17088 heartbeater.cc:347] Connected to a master server at 127.16.85.190:42017
I20240502 07:58:06.550408 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 07:58:06.550573 17088 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:06.551012 17088 heartbeater.cc:511] Master 127.16.85.190:42017 requested a full tablet report, sending...
I20240502 07:58:06.551782 16769 ts_manager.cc:194] Registered new tserver with Master: 6e77183c846f4c3dbed2acb777d12f9a (127.16.85.130:43625)
I20240502 07:58:06.552099 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.131:0
--local_ip_for_outbound_sockets=127.16.85.131
--webserver_interface=127.16.85.131
--webserver_port=0
--tserver_master_addrs=127.16.85.190:42017
--builtin_ntp_servers=127.16.85.148:33259
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_cache_size_limit_mb=10
--consensus_rpc_timeout_ms=30000 with env {}
I20240502 07:58:06.552280 16769 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.130:50315
W20240502 07:58:06.642818 17091 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:06.642992 17091 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:06.643045 17091 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:06.645722 17091 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:06.645817 17091 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.131
I20240502 07:58:06.648917 17091 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33259
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_cache_size_limit_mb=10
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.16.85.131
--webserver_port=0
--tserver_master_addrs=127.16.85.190:42017
--never_fsync=true
--heap_profile_path=/tmp/kudu.17091
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.131
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:06.649633 17091 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:06.650631 17091 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:06.657258 17097 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:06.657811 17099 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:06.658357 17096 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:06.659054 17091 server_base.cc:1034] running on GCE node
I20240502 07:58:06.659348 17091 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:06.659817 17091 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:06.660970 17091 hybrid_clock.cc:648] HybridClock initialized: now 1714636686660949 us; error 31 us; skew 500 ppm
I20240502 07:58:06.662470 17091 webserver.cc:460] Webserver started at http://127.16.85.131:42451/ using document root <none> and password file <none>
I20240502 07:58:06.662918 17091 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:06.662964 17091 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:06.663118 17091 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:06.664526 17091 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "537dd515c2ad432fa5edf12e19165b02"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.664965 17091 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "537dd515c2ad432fa5edf12e19165b02"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.667769 17091 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:06.669766 17105 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:06.670648 17091 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:06.670729 17091 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "537dd515c2ad432fa5edf12e19165b02"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.670815 17091 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:06.690052 17091 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:06.690560 17091 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:06.690692 17091 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:06.691133 17091 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:06.691838 17091 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:06.691875 17091 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:06.691922 17091 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:06.691952 17091 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:06.700173 17091 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.131:45845
I20240502 07:58:06.700192 17218 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.131:45845 every 8 connection(s)
I20240502 07:58:06.700923 17091 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 07:58:06.706863 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 17091
I20240502 07:58:06.706957 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 07:58:06.707621 17219 heartbeater.cc:347] Connected to a master server at 127.16.85.190:42017
I20240502 07:58:06.707798 17219 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:06.708222 17219 heartbeater.cc:511] Master 127.16.85.190:42017 requested a full tablet report, sending...
I20240502 07:58:06.708642 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.132:0
--local_ip_for_outbound_sockets=127.16.85.132
--webserver_interface=127.16.85.132
--webserver_port=0
--tserver_master_addrs=127.16.85.190:42017
--builtin_ntp_servers=127.16.85.148:33259
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_cache_size_limit_mb=10
--consensus_rpc_timeout_ms=30000 with env {}
I20240502 07:58:06.709188 16769 ts_manager.cc:194] Registered new tserver with Master: 537dd515c2ad432fa5edf12e19165b02 (127.16.85.131:45845)
I20240502 07:58:06.709920 16769 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.131:57643
W20240502 07:58:06.797539 17222 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:06.797695 17222 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:06.797729 17222 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:06.800349 17222 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:06.800423 17222 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.132
I20240502 07:58:06.803320 17222 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33259
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_cache_size_limit_mb=10
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.132:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.16.85.132
--webserver_port=0
--tserver_master_addrs=127.16.85.190:42017
--never_fsync=true
--heap_profile_path=/tmp/kudu.17222
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.132
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:06.803939 17222 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:06.804798 17222 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:06.811121 17228 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:06.811690 17230 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:06.812178 17227 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:06.812695 17222 server_base.cc:1034] running on GCE node
I20240502 07:58:06.812979 17222 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:06.813407 17222 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:06.814543 17222 hybrid_clock.cc:648] HybridClock initialized: now 1714636686814536 us; error 21 us; skew 500 ppm
I20240502 07:58:06.816076 17222 webserver.cc:460] Webserver started at http://127.16.85.132:40833/ using document root <none> and password file <none>
I20240502 07:58:06.816504 17222 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:06.816553 17222 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:06.816740 17222 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:06.818122 17222 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "59a392bf01ff4623a620b2ebd22b504a"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.818513 17222 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "59a392bf01ff4623a620b2ebd22b504a"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.821286 17222 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.004s sys 0.000s
I20240502 07:58:06.823009 17236 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:06.823810 17222 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:06.823886 17222 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "59a392bf01ff4623a620b2ebd22b504a"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.823957 17222 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:06.836736 17222 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:06.837219 17222 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:06.837342 17222 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:06.837744 17222 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:06.838464 17222 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:06.838503 17222 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:06.838541 17222 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:06.838570 17222 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:06.845834 17222 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.132:39603
I20240502 07:58:06.845857 17349 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.132:39603 every 8 connection(s)
I20240502 07:58:06.846563 17222 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 07:58:06.853699 17350 heartbeater.cc:347] Connected to a master server at 127.16.85.190:42017
I20240502 07:58:06.853739 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 17222
I20240502 07:58:06.853824 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 07:58:06.853899 17350 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:06.854401 17350 heartbeater.cc:511] Master 127.16.85.190:42017 requested a full tablet report, sending...
I20240502 07:58:06.855322 16769 ts_manager.cc:194] Registered new tserver with Master: 59a392bf01ff4623a620b2ebd22b504a (127.16.85.132:39603)
I20240502 07:58:06.855747 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.133:0
--local_ip_for_outbound_sockets=127.16.85.133
--webserver_interface=127.16.85.133
--webserver_port=0
--tserver_master_addrs=127.16.85.190:42017
--builtin_ntp_servers=127.16.85.148:33259
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_cache_size_limit_mb=10
--consensus_rpc_timeout_ms=30000 with env {}
I20240502 07:58:06.855971 16769 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.132:53439
W20240502 07:58:06.948101 17353 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:06.948256 17353 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:06.948289 17353 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:06.951066 17353 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:06.951143 17353 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.133
I20240502 07:58:06.954008 17353 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33259
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_cache_size_limit_mb=10
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.133:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.16.85.133
--webserver_port=0
--tserver_master_addrs=127.16.85.190:42017
--never_fsync=true
--heap_profile_path=/tmp/kudu.17353
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.133
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:06.954653 17353 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:06.955528 17353 file_cache.cc:492] Constructed file cache file cache with capacity 419430
I20240502 07:58:06.962672 17353 server_base.cc:1034] running on GCE node
W20240502 07:58:06.963102 17359 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:06.963657 17358 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:06.963663 17361 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:06.963991 17353 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:06.964414 17353 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:06.965557 17353 hybrid_clock.cc:648] HybridClock initialized: now 1714636686965536 us; error 33 us; skew 500 ppm
I20240502 07:58:06.967177 17353 webserver.cc:460] Webserver started at http://127.16.85.133:41213/ using document root <none> and password file <none>
I20240502 07:58:06.967626 17353 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:06.967679 17353 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:06.967821 17353 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:06.969306 17353 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "c51707f2c6454649a8dde1e86396b657"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.969734 17353 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "c51707f2c6454649a8dde1e86396b657"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.972467 17353 fs_manager.cc:691] Time spent creating directory manager: real 0.002s user 0.003s sys 0.000s
I20240502 07:58:06.974331 17367 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:06.975162 17353 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:06.975243 17353 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "c51707f2c6454649a8dde1e86396b657"
format_stamp: "Formatted at 2024-05-02 07:58:06 on dist-test-slave-7df7"
I20240502 07:58:06.975312 17353 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:06.997640 17353 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:06.998147 17353 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:06.998268 17353 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:06.998677 17353 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:06.999446 17353 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:06.999490 17353 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:06.999542 17353 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:06.999573 17353 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:07.006474 17353 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.133:33963
I20240502 07:58:07.006517 17480 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.133:33963 every 8 connection(s)
I20240502 07:58:07.007256 17353 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 07:58:07.010500 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 17353
I20240502 07:58:07.010618 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceIgnoredTserversTest.Basic.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 07:58:07.014663 17481 heartbeater.cc:347] Connected to a master server at 127.16.85.190:42017
I20240502 07:58:07.014815 17481 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:07.015175 17481 heartbeater.cc:511] Master 127.16.85.190:42017 requested a full tablet report, sending...
I20240502 07:58:07.015890 16769 ts_manager.cc:194] Registered new tserver with Master: c51707f2c6454649a8dde1e86396b657 (127.16.85.133:33963)
I20240502 07:58:07.016332 16769 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.133:40369
I20240502 07:58:07.024253 16726 external_mini_cluster.cc:930] 5 TS(s) registered with all masters
I20240502 07:58:07.033581 16769 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:39058:
name: "TestTable"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
owner: "alice"
I20240502 07:58:07.047360 17415 tablet_service.cc:1467] Processing CreateTablet for tablet 157900fa50b948a79b98e1c950f9053a (DEFAULT_TABLE table=TestTable [id=2d95739867df4f39a3b14e3328ea40bc]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:07.048048 17415 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 157900fa50b948a79b98e1c950f9053a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:07.048419 17284 tablet_service.cc:1467] Processing CreateTablet for tablet 157900fa50b948a79b98e1c950f9053a (DEFAULT_TABLE table=TestTable [id=2d95739867df4f39a3b14e3328ea40bc]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:07.049153 17284 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 157900fa50b948a79b98e1c950f9053a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:07.052297 17498 tablet_bootstrap.cc:492] T 157900fa50b948a79b98e1c950f9053a P c51707f2c6454649a8dde1e86396b657: Bootstrap starting.
I20240502 07:58:07.053345 17499 tablet_bootstrap.cc:492] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a: Bootstrap starting.
I20240502 07:58:07.054411 17498 tablet_bootstrap.cc:654] T 157900fa50b948a79b98e1c950f9053a P c51707f2c6454649a8dde1e86396b657: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:07.054292 17022 tablet_service.cc:1467] Processing CreateTablet for tablet 157900fa50b948a79b98e1c950f9053a (DEFAULT_TABLE table=TestTable [id=2d95739867df4f39a3b14e3328ea40bc]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:07.054901 17499 tablet_bootstrap.cc:654] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:07.055032 17022 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 157900fa50b948a79b98e1c950f9053a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:07.055079 17498 log.cc:824] T 157900fa50b948a79b98e1c950f9053a P c51707f2c6454649a8dde1e86396b657: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:07.055492 17499 log.cc:824] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:07.056468 17498 tablet_bootstrap.cc:492] T 157900fa50b948a79b98e1c950f9053a P c51707f2c6454649a8dde1e86396b657: No bootstrap required, opened a new log
I20240502 07:58:07.056617 17498 ts_tablet_manager.cc:1367] T 157900fa50b948a79b98e1c950f9053a P c51707f2c6454649a8dde1e86396b657: Time spent bootstrapping tablet: real 0.004s user 0.004s sys 0.000s
I20240502 07:58:07.056851 17499 tablet_bootstrap.cc:492] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a: No bootstrap required, opened a new log
I20240502 07:58:07.056970 17499 ts_tablet_manager.cc:1367] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 07:58:07.059026 17502 tablet_bootstrap.cc:492] T 157900fa50b948a79b98e1c950f9053a P 6e77183c846f4c3dbed2acb777d12f9a: Bootstrap starting.
I20240502 07:58:07.060149 17498 raft_consensus.cc:348] T 157900fa50b948a79b98e1c950f9053a P c51707f2c6454649a8dde1e86396b657 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51707f2c6454649a8dde1e86396b657" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 33963 } } peers { permanent_uuid: "59a392bf01ff4623a620b2ebd22b504a" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39603 } } peers { permanent_uuid: "6e77183c846f4c3dbed2acb777d12f9a" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 43625 } }
I20240502 07:58:07.060184 17499 raft_consensus.cc:348] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51707f2c6454649a8dde1e86396b657" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 33963 } } peers { permanent_uuid: "59a392bf01ff4623a620b2ebd22b504a" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39603 } } peers { permanent_uuid: "6e77183c846f4c3dbed2acb777d12f9a" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 43625 } }
I20240502 07:58:07.060334 17498 raft_consensus.cc:374] T 157900fa50b948a79b98e1c950f9053a P c51707f2c6454649a8dde1e86396b657 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:07.060334 17499 raft_consensus.cc:374] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:07.060385 17498 raft_consensus.cc:729] T 157900fa50b948a79b98e1c950f9053a P c51707f2c6454649a8dde1e86396b657 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c51707f2c6454649a8dde1e86396b657, State: Initialized, Role: FOLLOWER
I20240502 07:58:07.060384 17499 raft_consensus.cc:729] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 59a392bf01ff4623a620b2ebd22b504a, State: Initialized, Role: FOLLOWER
I20240502 07:58:07.060612 17502 tablet_bootstrap.cc:654] T 157900fa50b948a79b98e1c950f9053a P 6e77183c846f4c3dbed2acb777d12f9a: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:07.060792 17499 consensus_queue.cc:260] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51707f2c6454649a8dde1e86396b657" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 33963 } } peers { permanent_uuid: "59a392bf01ff4623a620b2ebd22b504a" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39603 } } peers { permanent_uuid: "6e77183c846f4c3dbed2acb777d12f9a" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 43625 } }
I20240502 07:58:07.060787 17498 consensus_queue.cc:260] T 157900fa50b948a79b98e1c950f9053a P c51707f2c6454649a8dde1e86396b657 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51707f2c6454649a8dde1e86396b657" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 33963 } } peers { permanent_uuid: "59a392bf01ff4623a620b2ebd22b504a" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39603 } } peers { permanent_uuid: "6e77183c846f4c3dbed2acb777d12f9a" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 43625 } }
I20240502 07:58:07.061368 17502 log.cc:824] T 157900fa50b948a79b98e1c950f9053a P 6e77183c846f4c3dbed2acb777d12f9a: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:07.061520 17498 ts_tablet_manager.cc:1398] T 157900fa50b948a79b98e1c950f9053a P c51707f2c6454649a8dde1e86396b657: Time spent starting tablet: real 0.005s user 0.004s sys 0.000s
I20240502 07:58:07.061520 17499 ts_tablet_manager.cc:1398] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a: Time spent starting tablet: real 0.005s user 0.004s sys 0.000s
I20240502 07:58:07.061828 17350 heartbeater.cc:503] Master 127.16.85.190:42017 was elected leader, sending a full tablet report...
I20240502 07:58:07.062530 17502 tablet_bootstrap.cc:492] T 157900fa50b948a79b98e1c950f9053a P 6e77183c846f4c3dbed2acb777d12f9a: No bootstrap required, opened a new log
I20240502 07:58:07.062589 17481 heartbeater.cc:503] Master 127.16.85.190:42017 was elected leader, sending a full tablet report...
I20240502 07:58:07.062677 17502 ts_tablet_manager.cc:1367] T 157900fa50b948a79b98e1c950f9053a P 6e77183c846f4c3dbed2acb777d12f9a: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 07:58:07.065171 17502 raft_consensus.cc:348] T 157900fa50b948a79b98e1c950f9053a P 6e77183c846f4c3dbed2acb777d12f9a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51707f2c6454649a8dde1e86396b657" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 33963 } } peers { permanent_uuid: "59a392bf01ff4623a620b2ebd22b504a" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39603 } } peers { permanent_uuid: "6e77183c846f4c3dbed2acb777d12f9a" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 43625 } }
I20240502 07:58:07.065341 17502 raft_consensus.cc:374] T 157900fa50b948a79b98e1c950f9053a P 6e77183c846f4c3dbed2acb777d12f9a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:07.065400 17502 raft_consensus.cc:729] T 157900fa50b948a79b98e1c950f9053a P 6e77183c846f4c3dbed2acb777d12f9a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6e77183c846f4c3dbed2acb777d12f9a, State: Initialized, Role: FOLLOWER
I20240502 07:58:07.065693 17502 consensus_queue.cc:260] T 157900fa50b948a79b98e1c950f9053a P 6e77183c846f4c3dbed2acb777d12f9a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51707f2c6454649a8dde1e86396b657" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 33963 } } peers { permanent_uuid: "59a392bf01ff4623a620b2ebd22b504a" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39603 } } peers { permanent_uuid: "6e77183c846f4c3dbed2acb777d12f9a" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 43625 } }
I20240502 07:58:07.066150 17088 heartbeater.cc:503] Master 127.16.85.190:42017 was elected leader, sending a full tablet report...
I20240502 07:58:07.066262 17502 ts_tablet_manager.cc:1398] T 157900fa50b948a79b98e1c950f9053a P 6e77183c846f4c3dbed2acb777d12f9a: Time spent starting tablet: real 0.004s user 0.004s sys 0.000s
W20240502 07:58:07.097306 17351 tablet.cc:2369] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 07:58:07.258131 17482 tablet.cc:2369] T 157900fa50b948a79b98e1c950f9053a P c51707f2c6454649a8dde1e86396b657: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 07:58:07.280318 17504 raft_consensus.cc:482] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:07.280479 17504 raft_consensus.cc:504] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51707f2c6454649a8dde1e86396b657" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 33963 } } peers { permanent_uuid: "59a392bf01ff4623a620b2ebd22b504a" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39603 } } peers { permanent_uuid: "6e77183c846f4c3dbed2acb777d12f9a" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 43625 } }
I20240502 07:58:07.281482 17504 leader_election.cc:288] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c51707f2c6454649a8dde1e86396b657 (127.16.85.133:33963), 6e77183c846f4c3dbed2acb777d12f9a (127.16.85.130:43625)
I20240502 07:58:07.284986 17042 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "157900fa50b948a79b98e1c950f9053a" candidate_uuid: "59a392bf01ff4623a620b2ebd22b504a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6e77183c846f4c3dbed2acb777d12f9a" is_pre_election: true
I20240502 07:58:07.284989 17435 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "157900fa50b948a79b98e1c950f9053a" candidate_uuid: "59a392bf01ff4623a620b2ebd22b504a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c51707f2c6454649a8dde1e86396b657" is_pre_election: true
I20240502 07:58:07.285226 17435 raft_consensus.cc:2454] T 157900fa50b948a79b98e1c950f9053a P c51707f2c6454649a8dde1e86396b657 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 59a392bf01ff4623a620b2ebd22b504a in term 0.
I20240502 07:58:07.285233 17042 raft_consensus.cc:2454] T 157900fa50b948a79b98e1c950f9053a P 6e77183c846f4c3dbed2acb777d12f9a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 59a392bf01ff4623a620b2ebd22b504a in term 0.
I20240502 07:58:07.285626 17238 leader_election.cc:302] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 59a392bf01ff4623a620b2ebd22b504a, 6e77183c846f4c3dbed2acb777d12f9a; no voters:
I20240502 07:58:07.285840 17504 raft_consensus.cc:2781] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:07.285902 17504 raft_consensus.cc:482] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:07.285930 17504 raft_consensus.cc:3037] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:07.286751 17504 raft_consensus.cc:504] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51707f2c6454649a8dde1e86396b657" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 33963 } } peers { permanent_uuid: "59a392bf01ff4623a620b2ebd22b504a" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39603 } } peers { permanent_uuid: "6e77183c846f4c3dbed2acb777d12f9a" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 43625 } }
I20240502 07:58:07.287065 17504 leader_election.cc:288] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [CANDIDATE]: Term 1 election: Requested vote from peers c51707f2c6454649a8dde1e86396b657 (127.16.85.133:33963), 6e77183c846f4c3dbed2acb777d12f9a (127.16.85.130:43625)
I20240502 07:58:07.287313 17435 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "157900fa50b948a79b98e1c950f9053a" candidate_uuid: "59a392bf01ff4623a620b2ebd22b504a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c51707f2c6454649a8dde1e86396b657"
I20240502 07:58:07.287351 17042 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "157900fa50b948a79b98e1c950f9053a" candidate_uuid: "59a392bf01ff4623a620b2ebd22b504a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6e77183c846f4c3dbed2acb777d12f9a"
I20240502 07:58:07.287405 17435 raft_consensus.cc:3037] T 157900fa50b948a79b98e1c950f9053a P c51707f2c6454649a8dde1e86396b657 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:07.287438 17042 raft_consensus.cc:3037] T 157900fa50b948a79b98e1c950f9053a P 6e77183c846f4c3dbed2acb777d12f9a [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:07.288224 17042 raft_consensus.cc:2454] T 157900fa50b948a79b98e1c950f9053a P 6e77183c846f4c3dbed2acb777d12f9a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 59a392bf01ff4623a620b2ebd22b504a in term 1.
I20240502 07:58:07.288233 17435 raft_consensus.cc:2454] T 157900fa50b948a79b98e1c950f9053a P c51707f2c6454649a8dde1e86396b657 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 59a392bf01ff4623a620b2ebd22b504a in term 1.
I20240502 07:58:07.288444 17238 leader_election.cc:302] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 59a392bf01ff4623a620b2ebd22b504a, 6e77183c846f4c3dbed2acb777d12f9a; no voters:
I20240502 07:58:07.288589 17504 raft_consensus.cc:2781] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:07.288821 17504 raft_consensus.cc:686] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [term 1 LEADER]: Becoming Leader. State: Replica: 59a392bf01ff4623a620b2ebd22b504a, State: Running, Role: LEADER
I20240502 07:58:07.289085 17504 consensus_queue.cc:237] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51707f2c6454649a8dde1e86396b657" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 33963 } } peers { permanent_uuid: "59a392bf01ff4623a620b2ebd22b504a" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39603 } } peers { permanent_uuid: "6e77183c846f4c3dbed2acb777d12f9a" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 43625 } }
I20240502 07:58:07.292037 16769 catalog_manager.cc:5484] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a reported cstate change: term changed from 0 to 1, leader changed from <none> to 59a392bf01ff4623a620b2ebd22b504a (127.16.85.132). New cstate: current_term: 1 leader_uuid: "59a392bf01ff4623a620b2ebd22b504a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c51707f2c6454649a8dde1e86396b657" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 33963 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "59a392bf01ff4623a620b2ebd22b504a" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39603 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6e77183c846f4c3dbed2acb777d12f9a" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 43625 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:07.294453 17089 tablet.cc:2369] T 157900fa50b948a79b98e1c950f9053a P 6e77183c846f4c3dbed2acb777d12f9a: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 07:58:07.333869 16726 external_mini_cluster.cc:930] 5 TS(s) registered with all masters
I20240502 07:58:07.335234 16726 ts_itest-base.cc:233] Waiting for 0 tablets on tserver de577719e9174df89ddd3e411320a1e8 to finish bootstrapping
I20240502 07:58:07.340394 16726 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 6e77183c846f4c3dbed2acb777d12f9a to finish bootstrapping
I20240502 07:58:07.344205 16726 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 537dd515c2ad432fa5edf12e19165b02 to finish bootstrapping
I20240502 07:58:07.348579 16726 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 59a392bf01ff4623a620b2ebd22b504a to finish bootstrapping
I20240502 07:58:07.352269 16726 ts_itest-base.cc:233] Waiting for 1 tablets on tserver c51707f2c6454649a8dde1e86396b657 to finish bootstrapping
I20240502 07:58:07.406268 16957 heartbeater.cc:503] Master 127.16.85.190:42017 was elected leader, sending a full tablet report...
I20240502 07:58:07.506915 16891 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:07.506930 17022 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:07.507184 17153 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:07.508277 17284 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:07.509518 17415 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:07.684989 17504 consensus_queue.cc:1035] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [LEADER]: Connected to new peer: Peer: permanent_uuid: "c51707f2c6454649a8dde1e86396b657" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 33963 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:07.690057 17508 consensus_queue.cc:1035] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a [LEADER]: Connected to new peer: Peer: permanent_uuid: "6e77183c846f4c3dbed2acb777d12f9a" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 43625 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:07.700934 17153 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:07.701908 16891 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:07.709595 17284 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:07.709687 17022 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:07.712265 17415 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:07.718036 17219 heartbeater.cc:503] Master 127.16.85.190:42017 was elected leader, sending a full tablet report...
I20240502 07:58:07.889927 17284 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:07.890164 17153 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:07.890823 17022 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:07.898348 17415 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:07.901279 16891 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:07.964208 17153 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:07.965255 17022 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:07.965750 17284 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:07.966493 17415 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:07.967594 16891 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.054157 17153 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.054740 17284 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:08.056365 17415 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.056946 16891 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.064313 17022 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.209836 16768 ts_manager.cc:292] Set tserver state for de577719e9174df89ddd3e411320a1e8 to MAINTENANCE_MODE
I20240502 07:58:08.376001 17153 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.376376 17284 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:08.376823 17022 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.377313 17415 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.386503 16891 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.458904 17153 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.458930 17284 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:08.459000 17415 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.460249 17022 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.460950 16891 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.555668 17284 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:08.555932 17022 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.556941 17153 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.559361 16891 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.574952 17415 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.633584 17153 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.633941 17022 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.634088 17284 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:08.634780 16891 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.635257 17415 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.671036 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 16960
W20240502 07:58:08.714890 17238 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.130:43625: connect: Connection refused (error 111)
W20240502 07:58:08.715572 17238 consensus_peers.cc:485] T 157900fa50b948a79b98e1c950f9053a P 59a392bf01ff4623a620b2ebd22b504a -> Peer 6e77183c846f4c3dbed2acb777d12f9a (127.16.85.130:43625): Couldn't send request to peer 6e77183c846f4c3dbed2acb777d12f9a. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.130:43625: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:08.830629 17153 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.831921 17284 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:08.834362 17415 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.835388 16891 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.896781 17153 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.897214 17415 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.898491 16891 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.905418 17284 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:08.995728 17284 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:08.996325 17153 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:08.997225 17415 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:09.001658 16891 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:09.070335 17153 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:09.071487 17284 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:09.072026 17415 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:09.082336 16891 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:09.106243 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 16829
I20240502 07:58:09.111336 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 17091
I20240502 07:58:09.116135 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 17222
I20240502 07:58:09.121151 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 17353
I20240502 07:58:09.126009 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 16738
2024-05-02T07:58:09Z chronyd exiting
[ OK ] RebalanceIgnoredTserversTest.Basic (3125 ms)
[----------] 1 test from RebalanceIgnoredTserversTest (3125 ms total)
[----------] 1 test from RebalanceStartCriteriaTest
[ RUN ] RebalanceStartCriteriaTest.TabletServerIsDown/1
I20240502 07:58:09.147166 16726 test_util.cc:268] Using random seed: -154761959
I20240502 07:58:09.147922 16726 ts_itest-base.cc:112] Starting cluster with:
I20240502 07:58:09.147981 16726 ts_itest-base.cc:113] --------------
I20240502 07:58:09.147991 16726 ts_itest-base.cc:114] 5 tablet servers
I20240502 07:58:09.148020 16726 ts_itest-base.cc:115] 3 replicas per TS
I20240502 07:58:09.148029 16726 ts_itest-base.cc:116] --------------
2024-05-02T07:58:09Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T07:58:09Z Disabled control of system clock
I20240502 07:58:09.155617 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.16.85.190:38327
--webserver_interface=127.16.85.190
--webserver_port=0
--builtin_ntp_servers=127.16.85.148:43481
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.16.85.190:38327
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false with env {}
W20240502 07:58:09.251356 17951 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:09.251582 17951 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:09.251643 17951 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:09.254318 17951 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:09.254376 17951 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 07:58:09.254407 17951 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:09.254443 17951 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 07:58:09.254472 17951 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 07:58:09.257642 17951 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:43481
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.16.85.190:38327
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.16.85.190:38327
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.16.85.190
--webserver_port=0
--never_fsync=true
--heap_profile_path=/tmp/kudu.17951
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:09.258437 17951 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:09.259542 17951 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:09.266921 17957 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:09.267484 17959 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:09.267627 17956 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:09.268210 17951 server_base.cc:1034] running on GCE node
I20240502 07:58:09.268695 17951 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:09.269580 17951 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:09.270752 17951 hybrid_clock.cc:648] HybridClock initialized: now 1714636689270730 us; error 36 us; skew 500 ppm
I20240502 07:58:09.272420 17951 webserver.cc:460] Webserver started at http://127.16.85.190:41711/ using document root <none> and password file <none>
I20240502 07:58:09.272994 17951 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:09.273095 17951 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:09.273276 17951 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:09.274735 17951 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "336c6f1ecf8b41a49209b59cb49b9f1c"
format_stamp: "Formatted at 2024-05-02 07:58:09 on dist-test-slave-7df7"
I20240502 07:58:09.275106 17951 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "336c6f1ecf8b41a49209b59cb49b9f1c"
format_stamp: "Formatted at 2024-05-02 07:58:09 on dist-test-slave-7df7"
I20240502 07:58:09.278198 17951 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.001s sys 0.003s
I20240502 07:58:09.280238 17965 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:09.281172 17951 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
I20240502 07:58:09.281297 17951 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
uuid: "336c6f1ecf8b41a49209b59cb49b9f1c"
format_stamp: "Formatted at 2024-05-02 07:58:09 on dist-test-slave-7df7"
I20240502 07:58:09.281376 17951 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:09.289944 17951 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:09.290464 17951 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:09.290621 17951 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:09.296037 17951 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.190:38327
I20240502 07:58:09.296048 18017 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.190:38327 every 8 connection(s)
I20240502 07:58:09.296957 17951 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 07:58:09.299217 18018 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:09.300272 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 17951
I20240502 07:58:09.300391 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 07:58:09.304636 18018 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c: Bootstrap starting.
I20240502 07:58:09.306752 18018 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:09.307493 18018 log.cc:824] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:09.308961 18018 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c: No bootstrap required, opened a new log
I20240502 07:58:09.311257 18018 raft_consensus.cc:348] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "336c6f1ecf8b41a49209b59cb49b9f1c" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 38327 } }
I20240502 07:58:09.311412 18018 raft_consensus.cc:374] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:09.311447 18018 raft_consensus.cc:729] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 336c6f1ecf8b41a49209b59cb49b9f1c, State: Initialized, Role: FOLLOWER
I20240502 07:58:09.311777 18018 consensus_queue.cc:260] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "336c6f1ecf8b41a49209b59cb49b9f1c" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 38327 } }
I20240502 07:58:09.311872 18018 raft_consensus.cc:388] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:09.311911 18018 raft_consensus.cc:482] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:09.311980 18018 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:09.312729 18018 raft_consensus.cc:504] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "336c6f1ecf8b41a49209b59cb49b9f1c" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 38327 } }
I20240502 07:58:09.313030 18018 leader_election.cc:302] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 336c6f1ecf8b41a49209b59cb49b9f1c; no voters:
I20240502 07:58:09.313230 18018 leader_election.cc:288] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:09.313426 18023 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:09.313618 18023 raft_consensus.cc:686] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [term 1 LEADER]: Becoming Leader. State: Replica: 336c6f1ecf8b41a49209b59cb49b9f1c, State: Running, Role: LEADER
I20240502 07:58:09.313882 18023 consensus_queue.cc:237] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "336c6f1ecf8b41a49209b59cb49b9f1c" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 38327 } }
I20240502 07:58:09.314203 18018 sys_catalog.cc:564] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [sys.catalog]: configured and running, proceeding with master startup.
I20240502 07:58:09.315207 18025 sys_catalog.cc:455] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 336c6f1ecf8b41a49209b59cb49b9f1c. Latest consensus state: current_term: 1 leader_uuid: "336c6f1ecf8b41a49209b59cb49b9f1c" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "336c6f1ecf8b41a49209b59cb49b9f1c" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 38327 } } }
I20240502 07:58:09.315323 18025 sys_catalog.cc:458] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [sys.catalog]: This master's current role is: LEADER
I20240502 07:58:09.315608 18032 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 07:58:09.315582 18024 sys_catalog.cc:455] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "336c6f1ecf8b41a49209b59cb49b9f1c" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "336c6f1ecf8b41a49209b59cb49b9f1c" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 38327 } } }
I20240502 07:58:09.315708 18024 sys_catalog.cc:458] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c [sys.catalog]: This master's current role is: LEADER
I20240502 07:58:09.318032 18032 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 07:58:09.322615 18032 catalog_manager.cc:1347] Generated new cluster ID: bd269df040ee49979e1a2776b3c99632
I20240502 07:58:09.322672 18032 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 07:58:09.332814 18032 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 07:58:09.333534 18032 catalog_manager.cc:1504] Loading token signing keys...
I20240502 07:58:09.338986 18032 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 336c6f1ecf8b41a49209b59cb49b9f1c: Generated new TSK 0
I20240502 07:58:09.339629 18032 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 07:58:09.342720 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.129:0
--local_ip_for_outbound_sockets=127.16.85.129
--webserver_interface=127.16.85.129
--webserver_port=0
--tserver_master_addrs=127.16.85.190:38327
--builtin_ntp_servers=127.16.85.148:43481
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 07:58:09.438076 18042 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:09.438266 18042 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:09.438303 18042 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:09.441094 18042 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:09.441145 18042 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:09.441212 18042 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.129
I20240502 07:58:09.444190 18042 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:43481
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.16.85.129
--webserver_port=0
--tserver_master_addrs=127.16.85.190:38327
--never_fsync=true
--heap_profile_path=/tmp/kudu.18042
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.129
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:09.444972 18042 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:09.445963 18042 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:09.453884 18047 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:09.453951 18048 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:09.454735 18050 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:09.455291 18042 server_base.cc:1034] running on GCE node
I20240502 07:58:09.455603 18042 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:09.456058 18042 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:09.457193 18042 hybrid_clock.cc:648] HybridClock initialized: now 1714636689457175 us; error 31 us; skew 500 ppm
I20240502 07:58:09.458911 18042 webserver.cc:460] Webserver started at http://127.16.85.129:42317/ using document root <none> and password file <none>
I20240502 07:58:09.459432 18042 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:09.459486 18042 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:09.459671 18042 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:09.461236 18042 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "55c3fe58f9b64f5d97f45ee3be79cedc"
format_stamp: "Formatted at 2024-05-02 07:58:09 on dist-test-slave-7df7"
I20240502 07:58:09.461655 18042 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "55c3fe58f9b64f5d97f45ee3be79cedc"
format_stamp: "Formatted at 2024-05-02 07:58:09 on dist-test-slave-7df7"
I20240502 07:58:09.464541 18042 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 07:58:09.466367 18056 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:09.467276 18042 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:09.467372 18042 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "55c3fe58f9b64f5d97f45ee3be79cedc"
format_stamp: "Formatted at 2024-05-02 07:58:09 on dist-test-slave-7df7"
I20240502 07:58:09.467447 18042 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:09.495239 18042 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:09.495862 18042 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:09.496039 18042 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:09.496544 18042 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:09.497390 18042 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:09.497432 18042 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:09.497485 18042 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:09.497512 18042 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:09.504909 18042 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.129:45951
I20240502 07:58:09.504946 18169 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.129:45951 every 8 connection(s)
I20240502 07:58:09.505796 18042 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 07:58:09.507733 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 18042
I20240502 07:58:09.507831 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 07:58:09.509744 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.130:0
--local_ip_for_outbound_sockets=127.16.85.130
--webserver_interface=127.16.85.130
--webserver_port=0
--tserver_master_addrs=127.16.85.190:38327
--builtin_ntp_servers=127.16.85.148:43481
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:09.514117 18170 heartbeater.cc:347] Connected to a master server at 127.16.85.190:38327
I20240502 07:58:09.514338 18170 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:09.515007 18170 heartbeater.cc:511] Master 127.16.85.190:38327 requested a full tablet report, sending...
I20240502 07:58:09.516410 17982 ts_manager.cc:194] Registered new tserver with Master: 55c3fe58f9b64f5d97f45ee3be79cedc (127.16.85.129:45951)
I20240502 07:58:09.517534 17982 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.129:60129
W20240502 07:58:09.607955 18173 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:09.608137 18173 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:09.608172 18173 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:09.611033 18173 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:09.611095 18173 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:09.611147 18173 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.130
I20240502 07:58:09.614272 18173 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:43481
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.16.85.130
--webserver_port=0
--tserver_master_addrs=127.16.85.190:38327
--never_fsync=true
--heap_profile_path=/tmp/kudu.18173
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.130
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:09.615059 18173 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:09.616088 18173 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:09.623310 18181 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:09.623843 18179 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:09.624553 18178 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:09.625239 18173 server_base.cc:1034] running on GCE node
I20240502 07:58:09.625592 18173 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:09.626061 18173 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:09.627192 18173 hybrid_clock.cc:648] HybridClock initialized: now 1714636689627183 us; error 22 us; skew 500 ppm
I20240502 07:58:09.628824 18173 webserver.cc:460] Webserver started at http://127.16.85.130:38959/ using document root <none> and password file <none>
I20240502 07:58:09.629336 18173 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:09.629391 18173 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:09.629575 18173 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:09.631150 18173 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "c7df9fb5372345a3b1900865575310eb"
format_stamp: "Formatted at 2024-05-02 07:58:09 on dist-test-slave-7df7"
I20240502 07:58:09.631568 18173 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "c7df9fb5372345a3b1900865575310eb"
format_stamp: "Formatted at 2024-05-02 07:58:09 on dist-test-slave-7df7"
I20240502 07:58:09.634533 18173 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 07:58:09.636410 18187 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:09.637343 18173 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:09.637468 18173 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "c7df9fb5372345a3b1900865575310eb"
format_stamp: "Formatted at 2024-05-02 07:58:09 on dist-test-slave-7df7"
I20240502 07:58:09.637558 18173 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:09.649245 18173 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:09.649730 18173 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:09.649890 18173 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:09.650362 18173 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:09.651152 18173 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:09.651196 18173 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:09.651268 18173 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:09.651293 18173 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:09.658672 18173 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.130:42389
I20240502 07:58:09.658694 18300 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.130:42389 every 8 connection(s)
I20240502 07:58:09.659543 18173 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 07:58:09.665169 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 18173
I20240502 07:58:09.665290 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 07:58:09.667440 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.131:0
--local_ip_for_outbound_sockets=127.16.85.131
--webserver_interface=127.16.85.131
--webserver_port=0
--tserver_master_addrs=127.16.85.190:38327
--builtin_ntp_servers=127.16.85.148:43481
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:09.667625 18301 heartbeater.cc:347] Connected to a master server at 127.16.85.190:38327
I20240502 07:58:09.667820 18301 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:09.668375 18301 heartbeater.cc:511] Master 127.16.85.190:38327 requested a full tablet report, sending...
I20240502 07:58:09.669502 17982 ts_manager.cc:194] Registered new tserver with Master: c7df9fb5372345a3b1900865575310eb (127.16.85.130:42389)
I20240502 07:58:09.670270 17982 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.130:35423
W20240502 07:58:09.765589 18304 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:09.765786 18304 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:09.765825 18304 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:09.768463 18304 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:09.768515 18304 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:09.768572 18304 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.131
I20240502 07:58:09.771898 18304 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:43481
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.16.85.131
--webserver_port=0
--tserver_master_addrs=127.16.85.190:38327
--never_fsync=true
--heap_profile_path=/tmp/kudu.18304
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.131
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:09.772694 18304 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:09.773692 18304 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:09.780932 18309 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:09.781634 18310 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:09.782332 18312 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:09.782912 18304 server_base.cc:1034] running on GCE node
I20240502 07:58:09.783257 18304 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:09.783795 18304 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:09.784950 18304 hybrid_clock.cc:648] HybridClock initialized: now 1714636689784917 us; error 54 us; skew 500 ppm
I20240502 07:58:09.786688 18304 webserver.cc:460] Webserver started at http://127.16.85.131:39549/ using document root <none> and password file <none>
I20240502 07:58:09.787195 18304 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:09.787247 18304 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:09.787396 18304 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:09.788983 18304 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "475a7f970da4436aaf8a1ffac1d893e2"
format_stamp: "Formatted at 2024-05-02 07:58:09 on dist-test-slave-7df7"
I20240502 07:58:09.789443 18304 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "475a7f970da4436aaf8a1ffac1d893e2"
format_stamp: "Formatted at 2024-05-02 07:58:09 on dist-test-slave-7df7"
I20240502 07:58:09.792469 18304 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 07:58:09.794485 18318 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:09.795384 18304 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:09.795486 18304 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "475a7f970da4436aaf8a1ffac1d893e2"
format_stamp: "Formatted at 2024-05-02 07:58:09 on dist-test-slave-7df7"
I20240502 07:58:09.795558 18304 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:09.826750 18304 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:09.827379 18304 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:09.827525 18304 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:09.828019 18304 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:09.828902 18304 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:09.828948 18304 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:09.829015 18304 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:09.829038 18304 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:09.836769 18304 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.131:45863
I20240502 07:58:09.836797 18431 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.131:45863 every 8 connection(s)
I20240502 07:58:09.837653 18304 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 07:58:09.844302 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 18304
I20240502 07:58:09.844416 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 07:58:09.846252 18432 heartbeater.cc:347] Connected to a master server at 127.16.85.190:38327
I20240502 07:58:09.846385 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.132:0
--local_ip_for_outbound_sockets=127.16.85.132
--webserver_interface=127.16.85.132
--webserver_port=0
--tserver_master_addrs=127.16.85.190:38327
--builtin_ntp_servers=127.16.85.148:43481
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:09.846482 18432 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:09.847095 18432 heartbeater.cc:511] Master 127.16.85.190:38327 requested a full tablet report, sending...
I20240502 07:58:09.848140 17982 ts_manager.cc:194] Registered new tserver with Master: 475a7f970da4436aaf8a1ffac1d893e2 (127.16.85.131:45863)
I20240502 07:58:09.848709 17982 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.131:57651
W20240502 07:58:09.944420 18435 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:09.944633 18435 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:09.944718 18435 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:09.947444 18435 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:09.947504 18435 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:09.947580 18435 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.132
I20240502 07:58:09.950969 18435 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:43481
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.132:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.16.85.132
--webserver_port=0
--tserver_master_addrs=127.16.85.190:38327
--never_fsync=true
--heap_profile_path=/tmp/kudu.18435
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.132
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:09.951891 18435 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:09.952945 18435 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:09.960466 18440 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:09.961095 18441 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:09.961813 18443 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:09.962379 18435 server_base.cc:1034] running on GCE node
I20240502 07:58:09.962714 18435 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:09.963208 18435 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:09.964363 18435 hybrid_clock.cc:648] HybridClock initialized: now 1714636689964350 us; error 24 us; skew 500 ppm
I20240502 07:58:09.966135 18435 webserver.cc:460] Webserver started at http://127.16.85.132:33189/ using document root <none> and password file <none>
I20240502 07:58:09.966663 18435 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:09.966715 18435 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:09.966871 18435 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:09.968824 18435 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "920c381b8f984e1daecd372649770391"
format_stamp: "Formatted at 2024-05-02 07:58:09 on dist-test-slave-7df7"
I20240502 07:58:09.970533 18435 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "920c381b8f984e1daecd372649770391"
format_stamp: "Formatted at 2024-05-02 07:58:09 on dist-test-slave-7df7"
I20240502 07:58:09.973786 18435 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 07:58:09.975768 18449 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:09.976637 18435 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:09.976773 18435 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "920c381b8f984e1daecd372649770391"
format_stamp: "Formatted at 2024-05-02 07:58:09 on dist-test-slave-7df7"
I20240502 07:58:09.976850 18435 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:09.990988 18435 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:09.991524 18435 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:09.991662 18435 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:09.992130 18435 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:09.992961 18435 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:09.993003 18435 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:09.993038 18435 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:09.993057 18435 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:10.000355 18435 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.132:43641
I20240502 07:58:10.000380 18562 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.132:43641 every 8 connection(s)
I20240502 07:58:10.001197 18435 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 07:58:10.002449 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 18435
I20240502 07:58:10.002555 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 07:58:10.004319 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.133:0
--local_ip_for_outbound_sockets=127.16.85.133
--webserver_interface=127.16.85.133
--webserver_port=0
--tserver_master_addrs=127.16.85.190:38327
--builtin_ntp_servers=127.16.85.148:43481
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:10.009426 18563 heartbeater.cc:347] Connected to a master server at 127.16.85.190:38327
I20240502 07:58:10.009670 18563 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:10.010360 18563 heartbeater.cc:511] Master 127.16.85.190:38327 requested a full tablet report, sending...
I20240502 07:58:10.011389 17982 ts_manager.cc:194] Registered new tserver with Master: 920c381b8f984e1daecd372649770391 (127.16.85.132:43641)
I20240502 07:58:10.012084 17982 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.132:56657
W20240502 07:58:10.103372 18566 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:10.103564 18566 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:10.103598 18566 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:10.106082 18566 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:10.106132 18566 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:10.106184 18566 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.133
I20240502 07:58:10.109292 18566 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:43481
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.133:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.16.85.133
--webserver_port=0
--tserver_master_addrs=127.16.85.190:38327
--never_fsync=true
--heap_profile_path=/tmp/kudu.18566
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.133
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:10.110122 18566 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:10.111176 18566 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:10.118901 18574 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:10.119378 18572 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:10.119518 18571 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:10.120211 18566 server_base.cc:1034] running on GCE node
I20240502 07:58:10.120579 18566 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:10.121096 18566 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:10.122231 18566 hybrid_clock.cc:648] HybridClock initialized: now 1714636690122215 us; error 28 us; skew 500 ppm
I20240502 07:58:10.130952 18566 webserver.cc:460] Webserver started at http://127.16.85.133:34751/ using document root <none> and password file <none>
I20240502 07:58:10.131500 18566 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:10.131564 18566 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:10.131738 18566 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:10.133299 18566 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "b89710d013864eabbe3fb3a45404c247"
format_stamp: "Formatted at 2024-05-02 07:58:10 on dist-test-slave-7df7"
I20240502 07:58:10.133711 18566 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "b89710d013864eabbe3fb3a45404c247"
format_stamp: "Formatted at 2024-05-02 07:58:10 on dist-test-slave-7df7"
I20240502 07:58:10.136694 18566 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.004s sys 0.000s
I20240502 07:58:10.138710 18580 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:10.139676 18566 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:10.139778 18566 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "b89710d013864eabbe3fb3a45404c247"
format_stamp: "Formatted at 2024-05-02 07:58:10 on dist-test-slave-7df7"
I20240502 07:58:10.139853 18566 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:10.155993 18566 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:10.156543 18566 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:10.156737 18566 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:10.157248 18566 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:10.158048 18566 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:10.158092 18566 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:10.158149 18566 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:10.158174 18566 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:10.165540 18566 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.133:38815
I20240502 07:58:10.165560 18693 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.133:38815 every 8 connection(s)
I20240502 07:58:10.166404 18566 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 07:58:10.169771 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 18566
I20240502 07:58:10.169941 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceStartCriteriaTest.TabletServerIsDown_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 07:58:10.174639 18694 heartbeater.cc:347] Connected to a master server at 127.16.85.190:38327
I20240502 07:58:10.174865 18694 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:10.175326 18694 heartbeater.cc:511] Master 127.16.85.190:38327 requested a full tablet report, sending...
I20240502 07:58:10.176182 17982 ts_manager.cc:194] Registered new tserver with Master: b89710d013864eabbe3fb3a45404c247 (127.16.85.133:38815)
I20240502 07:58:10.176682 17982 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.133:34729
I20240502 07:58:10.183669 16726 external_mini_cluster.cc:930] 5 TS(s) registered with all masters
I20240502 07:58:10.191561 17982 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:56044:
name: "TestTable"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
owner: "alice"
I20240502 07:58:10.207353 18497 tablet_service.cc:1467] Processing CreateTablet for tablet 02817874144c475e942899fd47595860 (DEFAULT_TABLE table=TestTable [id=e430810e00b7406491d069e29e92ad53]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:10.207934 18235 tablet_service.cc:1467] Processing CreateTablet for tablet 02817874144c475e942899fd47595860 (DEFAULT_TABLE table=TestTable [id=e430810e00b7406491d069e29e92ad53]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:10.208249 18497 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 02817874144c475e942899fd47595860. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:10.208299 18628 tablet_service.cc:1467] Processing CreateTablet for tablet 02817874144c475e942899fd47595860 (DEFAULT_TABLE table=TestTable [id=e430810e00b7406491d069e29e92ad53]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:10.208801 18235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 02817874144c475e942899fd47595860. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:10.209044 18628 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 02817874144c475e942899fd47595860. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:10.212848 18711 tablet_bootstrap.cc:492] T 02817874144c475e942899fd47595860 P 920c381b8f984e1daecd372649770391: Bootstrap starting.
I20240502 07:58:10.213861 18712 tablet_bootstrap.cc:492] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb: Bootstrap starting.
I20240502 07:58:10.215047 18711 tablet_bootstrap.cc:654] T 02817874144c475e942899fd47595860 P 920c381b8f984e1daecd372649770391: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:10.215276 18713 tablet_bootstrap.cc:492] T 02817874144c475e942899fd47595860 P b89710d013864eabbe3fb3a45404c247: Bootstrap starting.
I20240502 07:58:10.215785 18712 tablet_bootstrap.cc:654] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:10.215859 18711 log.cc:824] T 02817874144c475e942899fd47595860 P 920c381b8f984e1daecd372649770391: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:10.216398 18712 log.cc:824] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:10.216807 18713 tablet_bootstrap.cc:654] T 02817874144c475e942899fd47595860 P b89710d013864eabbe3fb3a45404c247: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:10.217331 18713 log.cc:824] T 02817874144c475e942899fd47595860 P b89710d013864eabbe3fb3a45404c247: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:10.217464 18711 tablet_bootstrap.cc:492] T 02817874144c475e942899fd47595860 P 920c381b8f984e1daecd372649770391: No bootstrap required, opened a new log
I20240502 07:58:10.217621 18711 ts_tablet_manager.cc:1367] T 02817874144c475e942899fd47595860 P 920c381b8f984e1daecd372649770391: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240502 07:58:10.217944 18712 tablet_bootstrap.cc:492] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb: No bootstrap required, opened a new log
I20240502 07:58:10.218086 18712 ts_tablet_manager.cc:1367] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.001s
I20240502 07:58:10.218626 18713 tablet_bootstrap.cc:492] T 02817874144c475e942899fd47595860 P b89710d013864eabbe3fb3a45404c247: No bootstrap required, opened a new log
I20240502 07:58:10.218762 18713 ts_tablet_manager.cc:1367] T 02817874144c475e942899fd47595860 P b89710d013864eabbe3fb3a45404c247: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 07:58:10.221168 18711 raft_consensus.cc:348] T 02817874144c475e942899fd47595860 P 920c381b8f984e1daecd372649770391 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "920c381b8f984e1daecd372649770391" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 43641 } } peers { permanent_uuid: "b89710d013864eabbe3fb3a45404c247" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 38815 } } peers { permanent_uuid: "c7df9fb5372345a3b1900865575310eb" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 42389 } }
I20240502 07:58:10.221354 18711 raft_consensus.cc:374] T 02817874144c475e942899fd47595860 P 920c381b8f984e1daecd372649770391 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:10.221271 18713 raft_consensus.cc:348] T 02817874144c475e942899fd47595860 P b89710d013864eabbe3fb3a45404c247 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "920c381b8f984e1daecd372649770391" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 43641 } } peers { permanent_uuid: "b89710d013864eabbe3fb3a45404c247" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 38815 } } peers { permanent_uuid: "c7df9fb5372345a3b1900865575310eb" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 42389 } }
I20240502 07:58:10.221405 18711 raft_consensus.cc:729] T 02817874144c475e942899fd47595860 P 920c381b8f984e1daecd372649770391 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 920c381b8f984e1daecd372649770391, State: Initialized, Role: FOLLOWER
I20240502 07:58:10.221415 18713 raft_consensus.cc:374] T 02817874144c475e942899fd47595860 P b89710d013864eabbe3fb3a45404c247 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:10.221458 18713 raft_consensus.cc:729] T 02817874144c475e942899fd47595860 P b89710d013864eabbe3fb3a45404c247 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b89710d013864eabbe3fb3a45404c247, State: Initialized, Role: FOLLOWER
I20240502 07:58:10.221361 18712 raft_consensus.cc:348] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "920c381b8f984e1daecd372649770391" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 43641 } } peers { permanent_uuid: "b89710d013864eabbe3fb3a45404c247" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 38815 } } peers { permanent_uuid: "c7df9fb5372345a3b1900865575310eb" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 42389 } }
I20240502 07:58:10.221504 18712 raft_consensus.cc:374] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:10.221546 18712 raft_consensus.cc:729] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c7df9fb5372345a3b1900865575310eb, State: Initialized, Role: FOLLOWER
I20240502 07:58:10.221832 18713 consensus_queue.cc:260] T 02817874144c475e942899fd47595860 P b89710d013864eabbe3fb3a45404c247 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "920c381b8f984e1daecd372649770391" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 43641 } } peers { permanent_uuid: "b89710d013864eabbe3fb3a45404c247" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 38815 } } peers { permanent_uuid: "c7df9fb5372345a3b1900865575310eb" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 42389 } }
I20240502 07:58:10.221846 18712 consensus_queue.cc:260] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "920c381b8f984e1daecd372649770391" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 43641 } } peers { permanent_uuid: "b89710d013864eabbe3fb3a45404c247" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 38815 } } peers { permanent_uuid: "c7df9fb5372345a3b1900865575310eb" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 42389 } }
I20240502 07:58:10.221836 18711 consensus_queue.cc:260] T 02817874144c475e942899fd47595860 P 920c381b8f984e1daecd372649770391 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "920c381b8f984e1daecd372649770391" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 43641 } } peers { permanent_uuid: "b89710d013864eabbe3fb3a45404c247" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 38815 } } peers { permanent_uuid: "c7df9fb5372345a3b1900865575310eb" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 42389 } }
I20240502 07:58:10.222363 18694 heartbeater.cc:503] Master 127.16.85.190:38327 was elected leader, sending a full tablet report...
I20240502 07:58:10.222543 18713 ts_tablet_manager.cc:1398] T 02817874144c475e942899fd47595860 P b89710d013864eabbe3fb3a45404c247: Time spent starting tablet: real 0.004s user 0.001s sys 0.003s
I20240502 07:58:10.222544 18711 ts_tablet_manager.cc:1398] T 02817874144c475e942899fd47595860 P 920c381b8f984e1daecd372649770391: Time spent starting tablet: real 0.005s user 0.002s sys 0.003s
I20240502 07:58:10.222544 18712 ts_tablet_manager.cc:1398] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb: Time spent starting tablet: real 0.004s user 0.002s sys 0.003s
I20240502 07:58:10.222831 18301 heartbeater.cc:503] Master 127.16.85.190:38327 was elected leader, sending a full tablet report...
I20240502 07:58:10.223855 18563 heartbeater.cc:503] Master 127.16.85.190:38327 was elected leader, sending a full tablet report...
W20240502 07:58:10.252041 18564 tablet.cc:2369] T 02817874144c475e942899fd47595860 P 920c381b8f984e1daecd372649770391: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 07:58:10.273906 18718 raft_consensus.cc:482] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:10.274125 18718 raft_consensus.cc:504] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "920c381b8f984e1daecd372649770391" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 43641 } } peers { permanent_uuid: "b89710d013864eabbe3fb3a45404c247" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 38815 } } peers { permanent_uuid: "c7df9fb5372345a3b1900865575310eb" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 42389 } }
I20240502 07:58:10.275270 18718 leader_election.cc:288] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 920c381b8f984e1daecd372649770391 (127.16.85.132:43641), b89710d013864eabbe3fb3a45404c247 (127.16.85.133:38815)
I20240502 07:58:10.278944 18517 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "02817874144c475e942899fd47595860" candidate_uuid: "c7df9fb5372345a3b1900865575310eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "920c381b8f984e1daecd372649770391" is_pre_election: true
I20240502 07:58:10.278944 18648 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "02817874144c475e942899fd47595860" candidate_uuid: "c7df9fb5372345a3b1900865575310eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b89710d013864eabbe3fb3a45404c247" is_pre_election: true
I20240502 07:58:10.279228 18648 raft_consensus.cc:2454] T 02817874144c475e942899fd47595860 P b89710d013864eabbe3fb3a45404c247 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c7df9fb5372345a3b1900865575310eb in term 0.
I20240502 07:58:10.279238 18517 raft_consensus.cc:2454] T 02817874144c475e942899fd47595860 P 920c381b8f984e1daecd372649770391 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c7df9fb5372345a3b1900865575310eb in term 0.
I20240502 07:58:10.279640 18189 leader_election.cc:302] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 920c381b8f984e1daecd372649770391, c7df9fb5372345a3b1900865575310eb; no voters:
I20240502 07:58:10.279882 18718 raft_consensus.cc:2781] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:10.279955 18718 raft_consensus.cc:482] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:10.280004 18718 raft_consensus.cc:3037] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:10.280956 18718 raft_consensus.cc:504] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "920c381b8f984e1daecd372649770391" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 43641 } } peers { permanent_uuid: "b89710d013864eabbe3fb3a45404c247" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 38815 } } peers { permanent_uuid: "c7df9fb5372345a3b1900865575310eb" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 42389 } }
I20240502 07:58:10.281224 18718 leader_election.cc:288] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [CANDIDATE]: Term 1 election: Requested vote from peers 920c381b8f984e1daecd372649770391 (127.16.85.132:43641), b89710d013864eabbe3fb3a45404c247 (127.16.85.133:38815)
I20240502 07:58:10.281510 18517 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "02817874144c475e942899fd47595860" candidate_uuid: "c7df9fb5372345a3b1900865575310eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "920c381b8f984e1daecd372649770391"
I20240502 07:58:10.281582 18648 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "02817874144c475e942899fd47595860" candidate_uuid: "c7df9fb5372345a3b1900865575310eb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b89710d013864eabbe3fb3a45404c247"
I20240502 07:58:10.281608 18517 raft_consensus.cc:3037] T 02817874144c475e942899fd47595860 P 920c381b8f984e1daecd372649770391 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:10.281711 18648 raft_consensus.cc:3037] T 02817874144c475e942899fd47595860 P b89710d013864eabbe3fb3a45404c247 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:10.282559 18517 raft_consensus.cc:2454] T 02817874144c475e942899fd47595860 P 920c381b8f984e1daecd372649770391 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c7df9fb5372345a3b1900865575310eb in term 1.
I20240502 07:58:10.282661 18648 raft_consensus.cc:2454] T 02817874144c475e942899fd47595860 P b89710d013864eabbe3fb3a45404c247 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c7df9fb5372345a3b1900865575310eb in term 1.
I20240502 07:58:10.282824 18189 leader_election.cc:302] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 920c381b8f984e1daecd372649770391, c7df9fb5372345a3b1900865575310eb; no voters:
I20240502 07:58:10.283001 18718 raft_consensus.cc:2781] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:10.283185 18718 raft_consensus.cc:686] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [term 1 LEADER]: Becoming Leader. State: Replica: c7df9fb5372345a3b1900865575310eb, State: Running, Role: LEADER
I20240502 07:58:10.283457 18718 consensus_queue.cc:237] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "920c381b8f984e1daecd372649770391" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 43641 } } peers { permanent_uuid: "b89710d013864eabbe3fb3a45404c247" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 38815 } } peers { permanent_uuid: "c7df9fb5372345a3b1900865575310eb" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 42389 } }
I20240502 07:58:10.286219 17982 catalog_manager.cc:5484] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb reported cstate change: term changed from 0 to 1, leader changed from <none> to c7df9fb5372345a3b1900865575310eb (127.16.85.130). New cstate: current_term: 1 leader_uuid: "c7df9fb5372345a3b1900865575310eb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "920c381b8f984e1daecd372649770391" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 43641 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b89710d013864eabbe3fb3a45404c247" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 38815 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c7df9fb5372345a3b1900865575310eb" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 42389 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:10.304580 16726 external_mini_cluster.cc:930] 5 TS(s) registered with all masters
I20240502 07:58:10.305809 16726 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 55c3fe58f9b64f5d97f45ee3be79cedc to finish bootstrapping
I20240502 07:58:10.311497 16726 ts_itest-base.cc:233] Waiting for 1 tablets on tserver c7df9fb5372345a3b1900865575310eb to finish bootstrapping
I20240502 07:58:10.315732 16726 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 475a7f970da4436aaf8a1ffac1d893e2 to finish bootstrapping
I20240502 07:58:10.320668 16726 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 920c381b8f984e1daecd372649770391 to finish bootstrapping
I20240502 07:58:10.324623 16726 ts_itest-base.cc:233] Waiting for 1 tablets on tserver b89710d013864eabbe3fb3a45404c247 to finish bootstrapping
I20240502 07:58:10.328495 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 18042
W20240502 07:58:10.410745 18302 tablet.cc:2369] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 07:58:10.417227 18695 tablet.cc:2369] T 02817874144c475e942899fd47595860 P b89710d013864eabbe3fb3a45404c247: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 07:58:10.504226 18366 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:10.505101 18628 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:10.505321 18497 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:10.505551 18235 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:10.572396 18628 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:10.572989 18366 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:10.573312 18497 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:10.573988 18235 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:10.655280 18366 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:10.657033 18628 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:10.657634 18235 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:10.663556 18497 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:10.676489 18718 consensus_queue.cc:1035] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [LEADER]: Connected to new peer: Peer: permanent_uuid: "b89710d013864eabbe3fb3a45404c247" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 38815 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:10.681968 18722 consensus_queue.cc:1035] T 02817874144c475e942899fd47595860 P c7df9fb5372345a3b1900865575310eb [LEADER]: Connected to new peer: Peer: permanent_uuid: "920c381b8f984e1daecd372649770391" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 43641 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:10.850548 18432 heartbeater.cc:503] Master 127.16.85.190:38327 was elected leader, sending a full tablet report...
I20240502 07:58:10.859614 18366 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:10.860333 18497 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:10.860891 18628 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:10.861217 18235 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:10.928313 18366 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:10.929930 18497 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:10.932950 18628 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:10.933046 18235 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:11.015311 18366 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:11.015818 18497 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:11.017612 18628 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:11.025663 18235 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:11.051407 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 18173
I20240502 07:58:11.057180 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 18304
I20240502 07:58:11.062299 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 18435
I20240502 07:58:11.067469 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 18566
I20240502 07:58:11.072949 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 17951
2024-05-02T07:58:11Z chronyd exiting
[ OK ] RebalanceStartCriteriaTest.TabletServerIsDown/1 (1947 ms)
[----------] 1 test from RebalanceStartCriteriaTest (1947 ms total)
[----------] 1 test from RebalanceParamTest
[ RUN ] RebalanceParamTest.Rebalance/1
I20240502 07:58:11.094291 16726 test_util.cc:268] Using random seed: -152814834
I20240502 07:58:11.094969 16726 ts_itest-base.cc:112] Starting cluster with:
I20240502 07:58:11.095011 16726 ts_itest-base.cc:113] --------------
I20240502 07:58:11.095021 16726 ts_itest-base.cc:114] 7 tablet servers
I20240502 07:58:11.095029 16726 ts_itest-base.cc:115] 1 replicas per TS
I20240502 07:58:11.095038 16726 ts_itest-base.cc:116] --------------
2024-05-02T07:58:11Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T07:58:11Z Disabled control of system clock
I20240502 07:58:11.102246 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.16.85.190:41469
--webserver_interface=127.16.85.190
--webserver_port=0
--builtin_ntp_servers=127.16.85.148:33323
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.16.85.190:41469
--allow_unsafe_replication_factor
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--tserver_unresponsive_timeout_ms=3000 with env {}
W20240502 07:58:11.199862 18914 flags.cc:425] Enabled unsafe flag: --allow_unsafe_replication_factor=true
W20240502 07:58:11.200127 18914 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:11.200187 18914 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:11.200260 18914 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:11.203060 18914 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:11.203136 18914 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 07:58:11.203176 18914 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:11.203213 18914 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 07:58:11.203253 18914 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 07:58:11.206600 18914 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33323
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
--allow_unsafe_replication_factor=true
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.16.85.190:41469
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.16.85.190:41469
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.16.85.190
--webserver_port=0
--never_fsync=true
--heap_profile_path=/tmp/kudu.18914
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:11.207463 18914 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:11.208511 18914 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:11.215009 18919 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:11.215672 18914 server_base.cc:1034] running on GCE node
W20240502 07:58:11.216118 18920 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:11.216612 18922 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:11.217078 18914 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:11.217876 18914 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:11.219022 18914 hybrid_clock.cc:648] HybridClock initialized: now 1714636691219008 us; error 25 us; skew 500 ppm
I20240502 07:58:11.220829 18914 webserver.cc:460] Webserver started at http://127.16.85.190:37657/ using document root <none> and password file <none>
I20240502 07:58:11.221323 18914 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:11.221369 18914 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:11.221513 18914 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:11.222968 18914 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "2229511e8632482d8a20ec64c9c6ea56"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:11.223356 18914 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "2229511e8632482d8a20ec64c9c6ea56"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:11.226297 18914 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.003s
I20240502 07:58:11.228190 18928 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:11.229107 18914 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:11.229194 18914 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
uuid: "2229511e8632482d8a20ec64c9c6ea56"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:11.229283 18914 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:11.249198 18914 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:11.249843 18914 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:11.249995 18914 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:11.255889 18914 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.190:41469
I20240502 07:58:11.255897 18980 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.190:41469 every 8 connection(s)
I20240502 07:58:11.256837 18914 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 07:58:11.259150 18981 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:11.263649 18981 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56: Bootstrap starting.
I20240502 07:58:11.265666 18981 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:11.266266 18981 log.cc:824] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:11.267112 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 18914
I20240502 07:58:11.267233 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 07:58:11.267818 18981 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56: No bootstrap required, opened a new log
I20240502 07:58:11.270673 18981 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2229511e8632482d8a20ec64c9c6ea56" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 41469 } }
I20240502 07:58:11.270836 18981 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:11.270885 18981 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2229511e8632482d8a20ec64c9c6ea56, State: Initialized, Role: FOLLOWER
I20240502 07:58:11.271265 18981 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2229511e8632482d8a20ec64c9c6ea56" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 41469 } }
I20240502 07:58:11.271389 18981 raft_consensus.cc:388] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:11.271449 18981 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:11.271562 18981 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:11.272186 18981 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2229511e8632482d8a20ec64c9c6ea56" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 41469 } }
I20240502 07:58:11.272503 18981 leader_election.cc:302] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 2229511e8632482d8a20ec64c9c6ea56; no voters:
I20240502 07:58:11.272832 18981 leader_election.cc:288] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:11.273188 18986 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:11.273365 18986 raft_consensus.cc:686] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [term 1 LEADER]: Becoming Leader. State: Replica: 2229511e8632482d8a20ec64c9c6ea56, State: Running, Role: LEADER
I20240502 07:58:11.273627 18986 consensus_queue.cc:237] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2229511e8632482d8a20ec64c9c6ea56" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 41469 } }
I20240502 07:58:11.273838 18981 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [sys.catalog]: configured and running, proceeding with master startup.
I20240502 07:58:11.275195 18988 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2229511e8632482d8a20ec64c9c6ea56. Latest consensus state: current_term: 1 leader_uuid: "2229511e8632482d8a20ec64c9c6ea56" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2229511e8632482d8a20ec64c9c6ea56" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 41469 } } }
I20240502 07:58:11.275305 18988 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [sys.catalog]: This master's current role is: LEADER
I20240502 07:58:11.276103 18987 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "2229511e8632482d8a20ec64c9c6ea56" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "2229511e8632482d8a20ec64c9c6ea56" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 41469 } } }
I20240502 07:58:11.276242 18987 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56 [sys.catalog]: This master's current role is: LEADER
W20240502 07:58:11.278053 19001 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20240502 07:58:11.278110 19001 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20240502 07:58:11.278177 18999 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 07:58:11.278802 18999 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 07:58:11.282985 18999 catalog_manager.cc:1347] Generated new cluster ID: cfc89bb8e6c747a5871180f6af564296
I20240502 07:58:11.283039 18999 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 07:58:11.292775 18999 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 07:58:11.293884 18999 catalog_manager.cc:1504] Loading token signing keys...
I20240502 07:58:11.313376 18999 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 2229511e8632482d8a20ec64c9c6ea56: Generated new TSK 0
I20240502 07:58:11.314241 18999 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 07:58:11.318521 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.129:0
--local_ip_for_outbound_sockets=127.16.85.129
--webserver_interface=127.16.85.129
--webserver_port=0
--tserver_master_addrs=127.16.85.190:41469
--builtin_ntp_servers=127.16.85.148:33323
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 07:58:11.416419 19005 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:11.416618 19005 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:11.416699 19005 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:11.419202 19005 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:11.419260 19005 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:11.419361 19005 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.129
I20240502 07:58:11.422622 19005 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33323
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.16.85.129
--webserver_port=0
--tserver_master_addrs=127.16.85.190:41469
--never_fsync=true
--heap_profile_path=/tmp/kudu.19005
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.129
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:11.423416 19005 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:11.424451 19005 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:11.431768 19011 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:11.431913 19010 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:11.432556 19013 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:11.433146 19005 server_base.cc:1034] running on GCE node
I20240502 07:58:11.433430 19005 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:11.433869 19005 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:11.435024 19005 hybrid_clock.cc:648] HybridClock initialized: now 1714636691435019 us; error 25 us; skew 500 ppm
I20240502 07:58:11.436610 19005 webserver.cc:460] Webserver started at http://127.16.85.129:44671/ using document root <none> and password file <none>
I20240502 07:58:11.437093 19005 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:11.437145 19005 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:11.437314 19005 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:11.438704 19005 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "dfc05317d0c04450af55ceabe36f6a99"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:11.439106 19005 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "dfc05317d0c04450af55ceabe36f6a99"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:11.442055 19005 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 07:58:11.443928 19019 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:11.444744 19005 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:11.444860 19005 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "dfc05317d0c04450af55ceabe36f6a99"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:11.444927 19005 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:11.460866 19005 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:11.461297 19005 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:11.461414 19005 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:11.461814 19005 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:11.462483 19005 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:11.462525 19005 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:11.462572 19005 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:11.462599 19005 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:11.469667 19005 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.129:44153
I20240502 07:58:11.469693 19132 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.129:44153 every 8 connection(s)
I20240502 07:58:11.470433 19005 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 07:58:11.473667 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 19005
I20240502 07:58:11.473762 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 07:58:11.475629 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.130:0
--local_ip_for_outbound_sockets=127.16.85.130
--webserver_interface=127.16.85.130
--webserver_port=0
--tserver_master_addrs=127.16.85.190:41469
--builtin_ntp_servers=127.16.85.148:33323
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:11.478384 19133 heartbeater.cc:347] Connected to a master server at 127.16.85.190:41469
I20240502 07:58:11.478571 19133 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:11.479100 19133 heartbeater.cc:511] Master 127.16.85.190:41469 requested a full tablet report, sending...
I20240502 07:58:11.480861 18945 ts_manager.cc:194] Registered new tserver with Master: dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153)
I20240502 07:58:11.481799 18945 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.129:46105
W20240502 07:58:11.563349 19136 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:11.563486 19136 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:11.563521 19136 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:11.566099 19136 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:11.566145 19136 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:11.566218 19136 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.130
I20240502 07:58:11.569238 19136 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33323
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.16.85.130
--webserver_port=0
--tserver_master_addrs=127.16.85.190:41469
--never_fsync=true
--heap_profile_path=/tmp/kudu.19136
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.130
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:11.569902 19136 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:11.570833 19136 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:11.576859 19141 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:11.577432 19144 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:11.578303 19142 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:11.578924 19136 server_base.cc:1034] running on GCE node
I20240502 07:58:11.579190 19136 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:11.579608 19136 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:11.580714 19136 hybrid_clock.cc:648] HybridClock initialized: now 1714636691580700 us; error 33 us; skew 500 ppm
I20240502 07:58:11.582056 19136 webserver.cc:460] Webserver started at http://127.16.85.130:43227/ using document root <none> and password file <none>
I20240502 07:58:11.582453 19136 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:11.582501 19136 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:11.582633 19136 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:11.583869 19136 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "3a68f86d2b9d44108fbb383fd0ec3118"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:11.584244 19136 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "3a68f86d2b9d44108fbb383fd0ec3118"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:11.586730 19136 fs_manager.cc:691] Time spent creating directory manager: real 0.002s user 0.004s sys 0.000s
I20240502 07:58:11.588306 19150 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:11.589103 19136 fs_manager.cc:725] Time spent opening block manager: real 0.001s user 0.002s sys 0.000s
I20240502 07:58:11.589221 19136 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "3a68f86d2b9d44108fbb383fd0ec3118"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:11.589285 19136 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:11.602069 19136 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:11.602486 19136 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:11.602602 19136 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:11.602984 19136 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:11.603613 19136 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:11.603657 19136 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:11.603701 19136 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:11.603729 19136 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:11.610369 19136 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.130:33077
I20240502 07:58:11.610420 19263 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.130:33077 every 8 connection(s)
I20240502 07:58:11.611092 19136 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 07:58:11.617569 19264 heartbeater.cc:347] Connected to a master server at 127.16.85.190:41469
I20240502 07:58:11.617729 19264 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:11.618093 19264 heartbeater.cc:511] Master 127.16.85.190:41469 requested a full tablet report, sending...
I20240502 07:58:11.618801 18945 ts_manager.cc:194] Registered new tserver with Master: 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130:33077)
I20240502 07:58:11.619247 18945 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.130:38801
I20240502 07:58:11.620745 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 19136
I20240502 07:58:11.620828 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 07:58:11.622292 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.131:0
--local_ip_for_outbound_sockets=127.16.85.131
--webserver_interface=127.16.85.131
--webserver_port=0
--tserver_master_addrs=127.16.85.190:41469
--builtin_ntp_servers=127.16.85.148:33323
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 07:58:11.716218 19267 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:11.716375 19267 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:11.716439 19267 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:11.719146 19267 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:11.719202 19267 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:11.719288 19267 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.131
I20240502 07:58:11.722563 19267 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33323
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.16.85.131
--webserver_port=0
--tserver_master_addrs=127.16.85.190:41469
--never_fsync=true
--heap_profile_path=/tmp/kudu.19267
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.131
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:11.723212 19267 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:11.724047 19267 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:11.730216 19275 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:11.730986 19272 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:11.731626 19273 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:11.732189 19267 server_base.cc:1034] running on GCE node
I20240502 07:58:11.732481 19267 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:11.732954 19267 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:11.734090 19267 hybrid_clock.cc:648] HybridClock initialized: now 1714636691734080 us; error 23 us; skew 500 ppm
I20240502 07:58:11.735531 19267 webserver.cc:460] Webserver started at http://127.16.85.131:39243/ using document root <none> and password file <none>
I20240502 07:58:11.736016 19267 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:11.736071 19267 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:11.736215 19267 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:11.737588 19267 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "cbc178a0a2f545658c5961645415eb25"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:11.737977 19267 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "cbc178a0a2f545658c5961645415eb25"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:11.740651 19267 fs_manager.cc:691] Time spent creating directory manager: real 0.002s user 0.001s sys 0.002s
I20240502 07:58:11.742401 19281 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:11.743187 19267 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.002s
I20240502 07:58:11.743281 19267 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "cbc178a0a2f545658c5961645415eb25"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:11.743345 19267 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:11.757452 19267 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:11.757884 19267 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:11.758011 19267 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:11.758431 19267 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:11.759126 19267 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:11.759169 19267 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:11.759218 19267 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:11.759251 19267 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:11.766605 19267 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.131:42907
I20240502 07:58:11.766634 19394 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.131:42907 every 8 connection(s)
I20240502 07:58:11.767385 19267 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 07:58:11.774608 19395 heartbeater.cc:347] Connected to a master server at 127.16.85.190:41469
I20240502 07:58:11.774832 19395 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:11.775252 19395 heartbeater.cc:511] Master 127.16.85.190:41469 requested a full tablet report, sending...
I20240502 07:58:11.776113 18945 ts_manager.cc:194] Registered new tserver with Master: cbc178a0a2f545658c5961645415eb25 (127.16.85.131:42907)
I20240502 07:58:11.776419 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 19267
I20240502 07:58:11.776490 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 07:58:11.776623 18945 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.131:44617
I20240502 07:58:11.778259 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.132:0
--local_ip_for_outbound_sockets=127.16.85.132
--webserver_interface=127.16.85.132
--webserver_port=0
--tserver_master_addrs=127.16.85.190:41469
--builtin_ntp_servers=127.16.85.148:33323
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 07:58:11.871691 19398 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:11.871850 19398 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:11.871928 19398 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:11.874783 19398 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:11.874842 19398 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:11.874927 19398 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.132
I20240502 07:58:11.878312 19398 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33323
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.132:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.16.85.132
--webserver_port=0
--tserver_master_addrs=127.16.85.190:41469
--never_fsync=true
--heap_profile_path=/tmp/kudu.19398
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.132
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:11.878990 19398 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:11.879890 19398 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:11.886456 19404 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:11.887116 19406 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:11.887629 19403 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:11.888196 19398 server_base.cc:1034] running on GCE node
I20240502 07:58:11.888459 19398 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:11.888937 19398 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:11.890085 19398 hybrid_clock.cc:648] HybridClock initialized: now 1714636691890068 us; error 30 us; skew 500 ppm
I20240502 07:58:11.891561 19398 webserver.cc:460] Webserver started at http://127.16.85.132:37199/ using document root <none> and password file <none>
I20240502 07:58:11.892016 19398 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:11.892066 19398 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:11.892208 19398 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:11.893718 19398 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "278943ee039c48988108fc425b366ff7"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:11.894098 19398 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "278943ee039c48988108fc425b366ff7"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:11.896835 19398 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.004s sys 0.000s
I20240502 07:58:11.898557 19412 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:11.899384 19398 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:11.899483 19398 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "278943ee039c48988108fc425b366ff7"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:11.899554 19398 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:11.915088 19398 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:11.915555 19398 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:11.915692 19398 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:11.916117 19398 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:11.916865 19398 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:11.916911 19398 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:11.916960 19398 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:11.916992 19398 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:11.924384 19398 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.132:44867
I20240502 07:58:11.924407 19525 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.132:44867 every 8 connection(s)
I20240502 07:58:11.925109 19398 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 07:58:11.932289 19526 heartbeater.cc:347] Connected to a master server at 127.16.85.190:41469
I20240502 07:58:11.932453 19526 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:11.932969 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 19398
I20240502 07:58:11.932981 19526 heartbeater.cc:511] Master 127.16.85.190:41469 requested a full tablet report, sending...
I20240502 07:58:11.933079 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 07:58:11.933852 18945 ts_manager.cc:194] Registered new tserver with Master: 278943ee039c48988108fc425b366ff7 (127.16.85.132:44867)
I20240502 07:58:11.934310 18945 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.132:54151
I20240502 07:58:11.935578 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.133:0
--local_ip_for_outbound_sockets=127.16.85.133
--webserver_interface=127.16.85.133
--webserver_port=0
--tserver_master_addrs=127.16.85.190:41469
--builtin_ntp_servers=127.16.85.148:33323
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 07:58:12.025182 19529 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:12.025347 19529 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:12.025388 19529 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:12.027896 19529 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:12.027947 19529 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:12.028000 19529 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.133
I20240502 07:58:12.031142 19529 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33323
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.133:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.16.85.133
--webserver_port=0
--tserver_master_addrs=127.16.85.190:41469
--never_fsync=true
--heap_profile_path=/tmp/kudu.19529
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.133
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:12.031764 19529 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:12.032614 19529 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:12.039000 19535 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:12.039572 19534 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:12.040274 19537 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:12.040948 19529 server_base.cc:1034] running on GCE node
I20240502 07:58:12.041218 19529 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:12.041602 19529 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:12.042742 19529 hybrid_clock.cc:648] HybridClock initialized: now 1714636692042724 us; error 28 us; skew 500 ppm
I20240502 07:58:12.044234 19529 webserver.cc:460] Webserver started at http://127.16.85.133:33717/ using document root <none> and password file <none>
I20240502 07:58:12.044678 19529 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:12.044746 19529 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:12.044899 19529 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:12.046219 19529 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "3a4ce39a207341a4883e10e934b839b4"
format_stamp: "Formatted at 2024-05-02 07:58:12 on dist-test-slave-7df7"
I20240502 07:58:12.046602 19529 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "3a4ce39a207341a4883e10e934b839b4"
format_stamp: "Formatted at 2024-05-02 07:58:12 on dist-test-slave-7df7"
I20240502 07:58:12.049279 19529 fs_manager.cc:691] Time spent creating directory manager: real 0.002s user 0.004s sys 0.000s
I20240502 07:58:12.051136 19543 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:12.051893 19529 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:12.051980 19529 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "3a4ce39a207341a4883e10e934b839b4"
format_stamp: "Formatted at 2024-05-02 07:58:12 on dist-test-slave-7df7"
I20240502 07:58:12.052049 19529 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:12.066332 19529 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:12.066773 19529 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:12.066913 19529 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:12.067322 19529 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:12.068001 19529 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:12.068042 19529 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:12.068084 19529 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:12.068112 19529 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:12.075371 19529 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.133:36251
I20240502 07:58:12.075404 19656 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.133:36251 every 8 connection(s)
I20240502 07:58:12.076084 19529 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 07:58:12.080363 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 19529
I20240502 07:58:12.080468 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 07:58:12.082387 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.134:0
--local_ip_for_outbound_sockets=127.16.85.134
--webserver_interface=127.16.85.134
--webserver_port=0
--tserver_master_addrs=127.16.85.190:41469
--builtin_ntp_servers=127.16.85.148:33323
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:12.083397 19657 heartbeater.cc:347] Connected to a master server at 127.16.85.190:41469
I20240502 07:58:12.083597 19657 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:12.083999 19657 heartbeater.cc:511] Master 127.16.85.190:41469 requested a full tablet report, sending...
I20240502 07:58:12.084827 18945 ts_manager.cc:194] Registered new tserver with Master: 3a4ce39a207341a4883e10e934b839b4 (127.16.85.133:36251)
I20240502 07:58:12.085350 18945 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.133:49013
W20240502 07:58:12.173610 19660 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:12.173784 19660 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:12.173821 19660 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:12.176780 19660 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:12.176831 19660 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:12.176896 19660 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.134
I20240502 07:58:12.180006 19660 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33323
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.134:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.16.85.134
--webserver_port=0
--tserver_master_addrs=127.16.85.190:41469
--never_fsync=true
--heap_profile_path=/tmp/kudu.19660
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.134
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:12.180727 19660 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:12.181645 19660 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:12.188810 19668 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:12.188807 19665 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:12.189321 19666 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:12.190044 19660 server_base.cc:1034] running on GCE node
I20240502 07:58:12.190369 19660 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:12.190833 19660 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:12.191972 19660 hybrid_clock.cc:648] HybridClock initialized: now 1714636692191953 us; error 33 us; skew 500 ppm
I20240502 07:58:12.193583 19660 webserver.cc:460] Webserver started at http://127.16.85.134:46783/ using document root <none> and password file <none>
I20240502 07:58:12.194067 19660 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:12.194118 19660 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:12.194267 19660 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:12.195683 19660 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/instance:
uuid: "74506b6a243b450398cf31bdabf3eccf"
format_stamp: "Formatted at 2024-05-02 07:58:12 on dist-test-slave-7df7"
I20240502 07:58:12.196084 19660 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/instance:
uuid: "74506b6a243b450398cf31bdabf3eccf"
format_stamp: "Formatted at 2024-05-02 07:58:12 on dist-test-slave-7df7"
I20240502 07:58:12.198891 19660 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.004s sys 0.000s
I20240502 07:58:12.200747 19674 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:12.201558 19660 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:12.201649 19660 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "74506b6a243b450398cf31bdabf3eccf"
format_stamp: "Formatted at 2024-05-02 07:58:12 on dist-test-slave-7df7"
I20240502 07:58:12.201721 19660 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:12.226128 19660 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:12.226632 19660 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:12.226769 19660 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:12.227211 19660 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:12.227978 19660 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:12.228021 19660 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:12.228075 19660 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:12.228099 19660 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:12.236423 19660 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.134:36697
I20240502 07:58:12.236464 19787 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.134:36697 every 8 connection(s)
I20240502 07:58:12.237300 19660 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 07:58:12.244853 19788 heartbeater.cc:347] Connected to a master server at 127.16.85.190:41469
I20240502 07:58:12.245049 19788 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:12.245508 19788 heartbeater.cc:511] Master 127.16.85.190:41469 requested a full tablet report, sending...
I20240502 07:58:12.246385 18945 ts_manager.cc:194] Registered new tserver with Master: 74506b6a243b450398cf31bdabf3eccf (127.16.85.134:36697)
I20240502 07:58:12.246886 18945 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.134:44273
I20240502 07:58:12.247418 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 19660
I20240502 07:58:12.247529 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240502 07:58:12.249445 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.135:0
--local_ip_for_outbound_sockets=127.16.85.135
--webserver_interface=127.16.85.135
--webserver_port=0
--tserver_master_addrs=127.16.85.190:41469
--builtin_ntp_servers=127.16.85.148:33323
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 07:58:12.355476 19791 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:12.355674 19791 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:12.355712 19791 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:12.358438 19791 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:12.358489 19791 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:12.358574 19791 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.135
I20240502 07:58:12.362252 19791 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33323
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.135:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.16.85.135
--webserver_port=0
--tserver_master_addrs=127.16.85.190:41469
--never_fsync=true
--heap_profile_path=/tmp/kudu.19791
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.135
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:12.363024 19791 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:12.364054 19791 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:12.372018 19799 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:12.372637 19796 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:12.373165 19797 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:12.373855 19791 server_base.cc:1034] running on GCE node
I20240502 07:58:12.374184 19791 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:12.374665 19791 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:12.375816 19791 hybrid_clock.cc:648] HybridClock initialized: now 1714636692375808 us; error 24 us; skew 500 ppm
I20240502 07:58:12.377502 19791 webserver.cc:460] Webserver started at http://127.16.85.135:45259/ using document root <none> and password file <none>
I20240502 07:58:12.378008 19791 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:12.378067 19791 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:12.378237 19791 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:12.379798 19791 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/instance:
uuid: "f81ce2156e854e0ab96f4b6fb2077714"
format_stamp: "Formatted at 2024-05-02 07:58:12 on dist-test-slave-7df7"
I20240502 07:58:12.380244 19791 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/instance:
uuid: "f81ce2156e854e0ab96f4b6fb2077714"
format_stamp: "Formatted at 2024-05-02 07:58:12 on dist-test-slave-7df7"
I20240502 07:58:12.383332 19791 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.001s sys 0.004s
I20240502 07:58:12.385159 19805 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:12.386015 19791 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:12.386118 19791 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "f81ce2156e854e0ab96f4b6fb2077714"
format_stamp: "Formatted at 2024-05-02 07:58:12 on dist-test-slave-7df7"
I20240502 07:58:12.386185 19791 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:12.418666 19791 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:12.419274 19791 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:12.419454 19791 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:12.419986 19791 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:12.420912 19791 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:12.420966 19791 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:12.421023 19791 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:12.421056 19791 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:12.428871 19791 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.135:45589
I20240502 07:58:12.428894 19918 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.135:45589 every 8 connection(s)
I20240502 07:58:12.429698 19791 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 07:58:12.434888 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 19791
I20240502 07:58:12.435024 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/instance
I20240502 07:58:12.437750 19919 heartbeater.cc:347] Connected to a master server at 127.16.85.190:41469
I20240502 07:58:12.437956 19919 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:12.438407 19919 heartbeater.cc:511] Master 127.16.85.190:41469 requested a full tablet report, sending...
I20240502 07:58:12.439220 18945 ts_manager.cc:194] Registered new tserver with Master: f81ce2156e854e0ab96f4b6fb2077714 (127.16.85.135:45589)
I20240502 07:58:12.439739 18945 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.135:41477
I20240502 07:58:12.448575 16726 external_mini_cluster.cc:930] 7 TS(s) registered with all masters
I20240502 07:58:12.456352 18945 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50010:
name: "TestTable"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
owner: "alice"
I20240502 07:58:12.471172 19853 tablet_service.cc:1467] Processing CreateTablet for tablet 3b5a6f09d531470cbca30aa50654dcbf (DEFAULT_TABLE table=TestTable [id=98be05a6c41d4bfaa8f82f9de6ef01a7]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:12.471907 19853 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b5a6f09d531470cbca30aa50654dcbf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:12.476419 19933 tablet_bootstrap.cc:492] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714: Bootstrap starting.
I20240502 07:58:12.478207 19933 tablet_bootstrap.cc:654] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:12.478865 19933 log.cc:824] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:12.480216 19933 tablet_bootstrap.cc:492] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714: No bootstrap required, opened a new log
I20240502 07:58:12.480381 19933 ts_tablet_manager.cc:1367] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 07:58:12.483573 19933 raft_consensus.cc:348] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } }
I20240502 07:58:12.483736 19933 raft_consensus.cc:374] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:12.483780 19933 raft_consensus.cc:729] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f81ce2156e854e0ab96f4b6fb2077714, State: Initialized, Role: FOLLOWER
I20240502 07:58:12.483841 19133 heartbeater.cc:503] Master 127.16.85.190:41469 was elected leader, sending a full tablet report...
I20240502 07:58:12.484164 19933 consensus_queue.cc:260] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } }
I20240502 07:58:12.484300 19933 raft_consensus.cc:388] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:12.484355 19933 raft_consensus.cc:482] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:12.484440 19933 raft_consensus.cc:3037] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:12.568331 19933 raft_consensus.cc:504] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } }
I20240502 07:58:12.568799 19933 leader_election.cc:302] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: f81ce2156e854e0ab96f4b6fb2077714; no voters:
I20240502 07:58:12.569149 19933 leader_election.cc:288] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:12.569280 19935 raft_consensus.cc:2781] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:12.569519 19935 raft_consensus.cc:686] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 1 LEADER]: Becoming Leader. State: Replica: f81ce2156e854e0ab96f4b6fb2077714, State: Running, Role: LEADER
I20240502 07:58:12.569867 19935 consensus_queue.cc:237] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } }
I20240502 07:58:12.570120 19933 ts_tablet_manager.cc:1398] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714: Time spent starting tablet: real 0.090s user 0.007s sys 0.000s
I20240502 07:58:12.570405 19919 heartbeater.cc:503] Master 127.16.85.190:41469 was elected leader, sending a full tablet report...
I20240502 07:58:12.574111 18945 catalog_manager.cc:5484] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 reported cstate change: term changed from 0 to 1, leader changed from <none> to f81ce2156e854e0ab96f4b6fb2077714 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "f81ce2156e854e0ab96f4b6fb2077714" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:12.585422 16726 external_mini_cluster.cc:930] 7 TS(s) registered with all masters
I20240502 07:58:12.586515 16726 ts_itest-base.cc:233] Waiting for 0 tablets on tserver dfc05317d0c04450af55ceabe36f6a99 to finish bootstrapping
I20240502 07:58:12.592070 16726 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 3a68f86d2b9d44108fbb383fd0ec3118 to finish bootstrapping
I20240502 07:58:12.597610 16726 ts_itest-base.cc:233] Waiting for 0 tablets on tserver cbc178a0a2f545658c5961645415eb25 to finish bootstrapping
I20240502 07:58:12.602386 16726 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 278943ee039c48988108fc425b366ff7 to finish bootstrapping
I20240502 07:58:12.607158 16726 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 3a4ce39a207341a4883e10e934b839b4 to finish bootstrapping
I20240502 07:58:12.611709 16726 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 74506b6a243b450398cf31bdabf3eccf to finish bootstrapping
I20240502 07:58:12.616756 16726 ts_itest-base.cc:233] Waiting for 1 tablets on tserver f81ce2156e854e0ab96f4b6fb2077714 to finish bootstrapping
I20240502 07:58:12.620738 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 19267
I20240502 07:58:12.620816 19264 heartbeater.cc:503] Master 127.16.85.190:41469 was elected leader, sending a full tablet report...
I20240502 07:58:12.626389 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 19398
I20240502 07:58:12.631549 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 19529
I20240502 07:58:12.636577 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 19660
I20240502 07:58:12.642122 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 19791
I20240502 07:58:16.398391 18945 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50010:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:16.409420 19198 tablet_service.cc:1467] Processing CreateTablet for tablet 4b9f9422ce124736b68e1933c95d4049 (DEFAULT_TABLE table=rebalance_test_table_0 [id=9a3705a54c6e44889df8c4a5749fffe2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:16.409454 19067 tablet_service.cc:1467] Processing CreateTablet for tablet f8550e00f0ba4b5ea52822f99add2cfe (DEFAULT_TABLE table=rebalance_test_table_0 [id=9a3705a54c6e44889df8c4a5749fffe2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:16.409454 19066 tablet_service.cc:1467] Processing CreateTablet for tablet 823e3d11a041417e90344de4350fee72 (DEFAULT_TABLE table=rebalance_test_table_0 [id=9a3705a54c6e44889df8c4a5749fffe2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:16.410379 19198 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4b9f9422ce124736b68e1933c95d4049. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:16.410409 19066 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 823e3d11a041417e90344de4350fee72. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:16.410724 19067 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f8550e00f0ba4b5ea52822f99add2cfe. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:16.414934 19955 tablet_bootstrap.cc:492] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118: Bootstrap starting.
I20240502 07:58:16.415508 19956 tablet_bootstrap.cc:492] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99: Bootstrap starting.
I20240502 07:58:16.416903 19955 tablet_bootstrap.cc:654] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:16.417287 19956 tablet_bootstrap.cc:654] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:16.417641 19955 log.cc:824] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:16.417908 19956 log.cc:824] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:16.419145 19956 tablet_bootstrap.cc:492] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99: No bootstrap required, opened a new log
I20240502 07:58:16.419153 19955 tablet_bootstrap.cc:492] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118: No bootstrap required, opened a new log
I20240502 07:58:16.419278 19955 ts_tablet_manager.cc:1367] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 07:58:16.419279 19956 ts_tablet_manager.cc:1367] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 07:58:16.422037 19956 raft_consensus.cc:348] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.422076 19955 raft_consensus.cc:348] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:16.422192 19956 raft_consensus.cc:374] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:16.422202 19955 raft_consensus.cc:374] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:16.422233 19956 raft_consensus.cc:729] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Initialized, Role: FOLLOWER
I20240502 07:58:16.422257 19955 raft_consensus.cc:729] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Initialized, Role: FOLLOWER
I20240502 07:58:16.422580 19955 consensus_queue.cc:260] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:16.422580 19956 consensus_queue.cc:260] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.422679 19956 raft_consensus.cc:388] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:16.422685 19955 raft_consensus.cc:388] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:16.422737 19955 raft_consensus.cc:482] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:16.422735 19956 raft_consensus.cc:482] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:16.422823 19956 raft_consensus.cc:3037] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:16.422834 19955 raft_consensus.cc:3037] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:16.423447 19955 raft_consensus.cc:504] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:16.423456 19956 raft_consensus.cc:504] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.423694 19955 leader_election.cc:302] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 3a68f86d2b9d44108fbb383fd0ec3118; no voters:
I20240502 07:58:16.423694 19956 leader_election.cc:302] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dfc05317d0c04450af55ceabe36f6a99; no voters:
I20240502 07:58:16.423925 19955 leader_election.cc:288] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:16.423923 19956 leader_election.cc:288] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:16.423987 19959 raft_consensus.cc:2781] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:16.424144 19959 raft_consensus.cc:686] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Becoming Leader. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Running, Role: LEADER
I20240502 07:58:16.424392 19959 consensus_queue.cc:237] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.424607 19955 ts_tablet_manager.cc:1398] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118: Time spent starting tablet: real 0.005s user 0.006s sys 0.000s
I20240502 07:58:16.424614 19956 ts_tablet_manager.cc:1398] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99: Time spent starting tablet: real 0.005s user 0.006s sys 0.000s
I20240502 07:58:16.424927 19956 tablet_bootstrap.cc:492] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99: Bootstrap starting.
I20240502 07:58:16.425715 19956 tablet_bootstrap.cc:654] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:16.426364 19960 raft_consensus.cc:2781] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:16.426535 19960 raft_consensus.cc:686] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Becoming Leader. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Running, Role: LEADER
I20240502 07:58:16.426764 19960 consensus_queue.cc:237] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:16.427033 18945 catalog_manager.cc:5484] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: term changed from 0 to 1, leader changed from <none> to dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:16.427138 19956 tablet_bootstrap.cc:492] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99: No bootstrap required, opened a new log
I20240502 07:58:16.427186 19956 ts_tablet_manager.cc:1367] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:16.427573 19956 raft_consensus.cc:348] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.427659 19956 raft_consensus.cc:374] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:16.427685 19956 raft_consensus.cc:729] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Initialized, Role: FOLLOWER
I20240502 07:58:16.427839 19956 consensus_queue.cc:260] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.427910 19956 raft_consensus.cc:388] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:16.427948 19956 raft_consensus.cc:482] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:16.427996 19956 raft_consensus.cc:3037] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:16.428694 19956 raft_consensus.cc:504] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.428814 19956 leader_election.cc:302] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dfc05317d0c04450af55ceabe36f6a99; no voters:
I20240502 07:58:16.428901 19956 leader_election.cc:288] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:16.429104 19956 ts_tablet_manager.cc:1398] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:16.429620 19959 raft_consensus.cc:2781] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:16.429697 19959 raft_consensus.cc:686] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Becoming Leader. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Running, Role: LEADER
I20240502 07:58:16.429793 19959 consensus_queue.cc:237] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.431113 18945 catalog_manager.cc:5484] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: term changed from 0 to 1, leader changed from <none> to dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:16.432155 18944 catalog_manager.cc:5484] T 4b9f9422ce124736b68e1933c95d4049 P 3a68f86d2b9d44108fbb383fd0ec3118 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } health_report { overall_health: HEALTHY } } }
W20240502 07:58:16.532472 19971 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:16.532749 19971 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:16.535418 19971 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_0'
INSERT report
rows total: 2000
time total: 73.8701 ms
time per row: 0.036935 ms
I20240502 07:58:16.648761 18938 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50010:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:16.653460 19198 tablet_service.cc:1467] Processing CreateTablet for tablet 52d9c7b102b24b48b81b8cd9642ecbdb (DEFAULT_TABLE table=rebalance_test_table_1 [id=cf7c2ac2aad3476c911268e8abb1e143]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:16.653568 19197 tablet_service.cc:1467] Processing CreateTablet for tablet af6d015cb2154a29970c5a7a4116efe9 (DEFAULT_TABLE table=rebalance_test_table_1 [id=cf7c2ac2aad3476c911268e8abb1e143]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:16.653615 19066 tablet_service.cc:1467] Processing CreateTablet for tablet 26330684e7454ddf983b85b66c99442e (DEFAULT_TABLE table=rebalance_test_table_1 [id=cf7c2ac2aad3476c911268e8abb1e143]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:16.653733 19198 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 52d9c7b102b24b48b81b8cd9642ecbdb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:16.653844 19066 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 26330684e7454ddf983b85b66c99442e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:16.654023 19197 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet af6d015cb2154a29970c5a7a4116efe9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:16.655400 19955 tablet_bootstrap.cc:492] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118: Bootstrap starting.
I20240502 07:58:16.655551 19956 tablet_bootstrap.cc:492] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99: Bootstrap starting.
I20240502 07:58:16.656219 19955 tablet_bootstrap.cc:654] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:16.656219 19956 tablet_bootstrap.cc:654] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:16.657248 19956 tablet_bootstrap.cc:492] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99: No bootstrap required, opened a new log
I20240502 07:58:16.657327 19956 ts_tablet_manager.cc:1367] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:16.657415 19955 tablet_bootstrap.cc:492] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118: No bootstrap required, opened a new log
I20240502 07:58:16.657485 19955 ts_tablet_manager.cc:1367] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:16.657713 19956 raft_consensus.cc:348] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.657801 19956 raft_consensus.cc:374] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:16.657840 19956 raft_consensus.cc:729] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Initialized, Role: FOLLOWER
I20240502 07:58:16.657946 19955 raft_consensus.cc:348] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:16.657959 19956 consensus_queue.cc:260] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.658025 19955 raft_consensus.cc:374] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:16.658033 19956 raft_consensus.cc:388] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:16.658067 19956 raft_consensus.cc:482] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:16.658071 19955 raft_consensus.cc:729] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Initialized, Role: FOLLOWER
I20240502 07:58:16.658118 19956 raft_consensus.cc:3037] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:16.658180 19955 consensus_queue.cc:260] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:16.658272 19955 raft_consensus.cc:388] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:16.658308 19955 raft_consensus.cc:482] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:16.658346 19955 raft_consensus.cc:3037] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:16.658881 19956 raft_consensus.cc:504] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.659018 19956 leader_election.cc:302] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dfc05317d0c04450af55ceabe36f6a99; no voters:
I20240502 07:58:16.659044 19955 raft_consensus.cc:504] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:16.659111 19956 leader_election.cc:288] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:16.659157 19955 leader_election.cc:302] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 3a68f86d2b9d44108fbb383fd0ec3118; no voters:
I20240502 07:58:16.659157 19962 raft_consensus.cc:2781] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:16.659260 19962 raft_consensus.cc:686] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Becoming Leader. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Running, Role: LEADER
I20240502 07:58:16.659276 19966 raft_consensus.cc:2781] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:16.659327 19956 ts_tablet_manager.cc:1398] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:16.659334 19966 raft_consensus.cc:686] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Becoming Leader. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Running, Role: LEADER
I20240502 07:58:16.659353 19962 consensus_queue.cc:237] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.659427 19966 consensus_queue.cc:237] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:16.659777 19955 leader_election.cc:288] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:16.660059 19955 ts_tablet_manager.cc:1398] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 07:58:16.660228 19955 tablet_bootstrap.cc:492] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118: Bootstrap starting.
I20240502 07:58:16.661247 18944 catalog_manager.cc:5484] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:16.661310 19955 tablet_bootstrap.cc:654] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:16.661509 18938 catalog_manager.cc:5484] T 26330684e7454ddf983b85b66c99442e P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: term changed from 0 to 1, leader changed from <none> to dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:16.662287 19955 tablet_bootstrap.cc:492] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118: No bootstrap required, opened a new log
I20240502 07:58:16.662360 19955 ts_tablet_manager.cc:1367] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:16.662724 19955 raft_consensus.cc:348] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:16.662797 19955 raft_consensus.cc:374] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:16.662823 19955 raft_consensus.cc:729] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Initialized, Role: FOLLOWER
I20240502 07:58:16.662943 19955 consensus_queue.cc:260] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:16.663023 19955 raft_consensus.cc:388] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:16.663069 19955 raft_consensus.cc:482] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:16.663108 19955 raft_consensus.cc:3037] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:16.663806 19955 raft_consensus.cc:504] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:16.663911 19955 leader_election.cc:302] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 3a68f86d2b9d44108fbb383fd0ec3118; no voters:
I20240502 07:58:16.664031 19966 raft_consensus.cc:2781] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:16.664104 19966 raft_consensus.cc:686] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Becoming Leader. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Running, Role: LEADER
I20240502 07:58:16.664191 19966 consensus_queue.cc:237] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:16.664497 19955 leader_election.cc:288] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:16.664775 19955 ts_tablet_manager.cc:1398] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:16.666023 18938 catalog_manager.cc:5484] T 52d9c7b102b24b48b81b8cd9642ecbdb P 3a68f86d2b9d44108fbb383fd0ec3118 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } health_report { overall_health: HEALTHY } } }
W20240502 07:58:16.760409 20012 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:16.760624 20012 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:16.763551 20012 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_1'
INSERT report
rows total: 2000
time total: 59.1459 ms
time per row: 0.0295729 ms
I20240502 07:58:16.857771 18944 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50010:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:16.862592 19066 tablet_service.cc:1467] Processing CreateTablet for tablet 72092a0eebee4b818fe377651dc77cbc (DEFAULT_TABLE table=rebalance_test_table_2 [id=19022d6115e942f19e6e4f2c19bc5100]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:16.862617 19198 tablet_service.cc:1467] Processing CreateTablet for tablet 95b1058e6dd5473ca41d534137ebb3eb (DEFAULT_TABLE table=rebalance_test_table_2 [id=19022d6115e942f19e6e4f2c19bc5100]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:16.862757 19067 tablet_service.cc:1467] Processing CreateTablet for tablet 12852f6b0d7242208c3f2beea408bc89 (DEFAULT_TABLE table=rebalance_test_table_2 [id=19022d6115e942f19e6e4f2c19bc5100]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:16.862881 19066 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 72092a0eebee4b818fe377651dc77cbc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:16.862906 19198 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 95b1058e6dd5473ca41d534137ebb3eb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:16.863051 19067 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 12852f6b0d7242208c3f2beea408bc89. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:16.864372 19955 tablet_bootstrap.cc:492] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118: Bootstrap starting.
I20240502 07:58:16.864890 19956 tablet_bootstrap.cc:492] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99: Bootstrap starting.
I20240502 07:58:16.865094 19955 tablet_bootstrap.cc:654] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:16.865720 19956 tablet_bootstrap.cc:654] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:16.866130 19955 tablet_bootstrap.cc:492] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118: No bootstrap required, opened a new log
I20240502 07:58:16.866218 19955 ts_tablet_manager.cc:1367] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:16.866592 19955 raft_consensus.cc:348] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:16.866686 19955 raft_consensus.cc:374] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:16.866725 19955 raft_consensus.cc:729] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Initialized, Role: FOLLOWER
I20240502 07:58:16.866770 19956 tablet_bootstrap.cc:492] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99: No bootstrap required, opened a new log
I20240502 07:58:16.866845 19956 ts_tablet_manager.cc:1367] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:16.866854 19955 consensus_queue.cc:260] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:16.866925 19955 raft_consensus.cc:388] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:16.866959 19955 raft_consensus.cc:482] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:16.866993 19955 raft_consensus.cc:3037] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:16.867245 19956 raft_consensus.cc:348] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.867327 19956 raft_consensus.cc:374] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:16.867358 19956 raft_consensus.cc:729] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Initialized, Role: FOLLOWER
I20240502 07:58:16.867458 19956 consensus_queue.cc:260] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.867544 19956 raft_consensus.cc:388] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:16.867579 19956 raft_consensus.cc:482] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:16.867623 19956 raft_consensus.cc:3037] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:16.867761 19955 raft_consensus.cc:504] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:16.867866 19955 leader_election.cc:302] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 3a68f86d2b9d44108fbb383fd0ec3118; no voters:
I20240502 07:58:16.868009 19960 raft_consensus.cc:2781] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:16.868077 19960 raft_consensus.cc:686] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Becoming Leader. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Running, Role: LEADER
I20240502 07:58:16.868180 19960 consensus_queue.cc:237] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:16.868396 19956 raft_consensus.cc:504] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.868517 19956 leader_election.cc:302] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dfc05317d0c04450af55ceabe36f6a99; no voters:
I20240502 07:58:16.868580 19955 leader_election.cc:288] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:16.868608 19956 leader_election.cc:288] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:16.868674 19962 raft_consensus.cc:2781] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:16.868753 19962 raft_consensus.cc:686] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Becoming Leader. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Running, Role: LEADER
I20240502 07:58:16.868866 19956 ts_tablet_manager.cc:1398] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 07:58:16.868867 19955 ts_tablet_manager.cc:1398] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 07:58:16.868849 19962 consensus_queue.cc:237] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.868992 19956 tablet_bootstrap.cc:492] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99: Bootstrap starting.
I20240502 07:58:16.869702 19956 tablet_bootstrap.cc:654] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:16.870178 18944 catalog_manager.cc:5484] T 72092a0eebee4b818fe377651dc77cbc P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: term changed from 0 to 1, leader changed from <none> to dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:16.870172 18938 catalog_manager.cc:5484] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:16.871405 19956 tablet_bootstrap.cc:492] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99: No bootstrap required, opened a new log
I20240502 07:58:16.871497 19956 ts_tablet_manager.cc:1367] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.001s
I20240502 07:58:16.871858 19956 raft_consensus.cc:348] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.871942 19956 raft_consensus.cc:374] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:16.871973 19956 raft_consensus.cc:729] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Initialized, Role: FOLLOWER
I20240502 07:58:16.872344 19956 consensus_queue.cc:260] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.872435 19956 raft_consensus.cc:388] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:16.872471 19956 raft_consensus.cc:482] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:16.872514 19956 raft_consensus.cc:3037] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:16.873184 19956 raft_consensus.cc:504] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.873327 19956 leader_election.cc:302] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dfc05317d0c04450af55ceabe36f6a99; no voters:
I20240502 07:58:16.873445 19956 leader_election.cc:288] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:16.873497 19963 raft_consensus.cc:2781] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:16.873610 19963 raft_consensus.cc:686] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Becoming Leader. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Running, Role: LEADER
I20240502 07:58:16.873661 19956 ts_tablet_manager.cc:1398] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:16.873703 19963 consensus_queue.cc:237] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:16.874739 18938 catalog_manager.cc:5484] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: term changed from 0 to 1, leader changed from <none> to dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } health_report { overall_health: HEALTHY } } }
W20240502 07:58:16.972613 20046 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:16.972815 20046 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:16.975345 20046 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_2'
INSERT report
rows total: 2000
time total: 64.4919 ms
time per row: 0.0322459 ms
I20240502 07:58:17.073060 18938 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50010:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:17.077533 19198 tablet_service.cc:1467] Processing CreateTablet for tablet 0cb1145d5f6343a1b205bb59221e95f4 (DEFAULT_TABLE table=rebalance_test_table_3 [id=44e95bee069c42d2b54ac8810ce8424f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:17.077572 19197 tablet_service.cc:1467] Processing CreateTablet for tablet b2c2c3940c1f49c89cf0219dd481fa0b (DEFAULT_TABLE table=rebalance_test_table_3 [id=44e95bee069c42d2b54ac8810ce8424f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:17.077631 19067 tablet_service.cc:1467] Processing CreateTablet for tablet 708e377a224e40deab16180eb8f32c82 (DEFAULT_TABLE table=rebalance_test_table_3 [id=44e95bee069c42d2b54ac8810ce8424f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:17.077804 19198 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0cb1145d5f6343a1b205bb59221e95f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:17.077831 19067 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 708e377a224e40deab16180eb8f32c82. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:17.077951 19197 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b2c2c3940c1f49c89cf0219dd481fa0b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:17.079347 19956 tablet_bootstrap.cc:492] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99: Bootstrap starting.
I20240502 07:58:17.079916 19955 tablet_bootstrap.cc:492] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118: Bootstrap starting.
I20240502 07:58:17.080196 19956 tablet_bootstrap.cc:654] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:17.080926 19955 tablet_bootstrap.cc:654] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:17.081272 19956 tablet_bootstrap.cc:492] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99: No bootstrap required, opened a new log
I20240502 07:58:17.081364 19956 ts_tablet_manager.cc:1367] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:17.081728 19956 raft_consensus.cc:348] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:17.081810 19956 raft_consensus.cc:374] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:17.081848 19956 raft_consensus.cc:729] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Initialized, Role: FOLLOWER
I20240502 07:58:17.081857 19955 tablet_bootstrap.cc:492] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118: No bootstrap required, opened a new log
I20240502 07:58:17.081925 19955 ts_tablet_manager.cc:1367] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:17.081976 19956 consensus_queue.cc:260] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:17.082067 19956 raft_consensus.cc:388] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:17.082106 19956 raft_consensus.cc:482] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:17.082152 19956 raft_consensus.cc:3037] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:17.082304 19955 raft_consensus.cc:348] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:17.082376 19955 raft_consensus.cc:374] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:17.082394 19955 raft_consensus.cc:729] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Initialized, Role: FOLLOWER
I20240502 07:58:17.082495 19955 consensus_queue.cc:260] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:17.082559 19955 raft_consensus.cc:388] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:17.082597 19955 raft_consensus.cc:482] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:17.082636 19955 raft_consensus.cc:3037] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:17.082891 19956 raft_consensus.cc:504] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:17.082989 19956 leader_election.cc:302] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dfc05317d0c04450af55ceabe36f6a99; no voters:
I20240502 07:58:17.083079 19956 leader_election.cc:288] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:17.083128 19962 raft_consensus.cc:2781] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:17.083201 19962 raft_consensus.cc:686] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Becoming Leader. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Running, Role: LEADER
I20240502 07:58:17.083303 19962 consensus_queue.cc:237] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:17.083348 19956 ts_tablet_manager.cc:1398] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 07:58:17.084157 19955 raft_consensus.cc:504] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:17.084291 19955 leader_election.cc:302] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 3a68f86d2b9d44108fbb383fd0ec3118; no voters:
I20240502 07:58:17.084398 19955 leader_election.cc:288] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:17.084447 19966 raft_consensus.cc:2781] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:17.084604 19966 raft_consensus.cc:686] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Becoming Leader. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Running, Role: LEADER
I20240502 07:58:17.084538 18938 catalog_manager.cc:5484] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: term changed from 0 to 1, leader changed from <none> to dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:17.084623 19955 ts_tablet_manager.cc:1398] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:17.084753 19966 consensus_queue.cc:237] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:17.084813 19955 tablet_bootstrap.cc:492] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118: Bootstrap starting.
I20240502 07:58:17.085732 19955 tablet_bootstrap.cc:654] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:17.086310 18944 catalog_manager.cc:5484] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:17.086776 19955 tablet_bootstrap.cc:492] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118: No bootstrap required, opened a new log
I20240502 07:58:17.086843 19955 ts_tablet_manager.cc:1367] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:17.087225 19955 raft_consensus.cc:348] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:17.087306 19955 raft_consensus.cc:374] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:17.087345 19955 raft_consensus.cc:729] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Initialized, Role: FOLLOWER
I20240502 07:58:17.087446 19955 consensus_queue.cc:260] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:17.087524 19955 raft_consensus.cc:388] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:17.087563 19955 raft_consensus.cc:482] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:17.087600 19955 raft_consensus.cc:3037] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:17.088352 19955 raft_consensus.cc:504] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:17.088457 19955 leader_election.cc:302] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 3a68f86d2b9d44108fbb383fd0ec3118; no voters:
I20240502 07:58:17.088534 19955 leader_election.cc:288] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:17.088582 19966 raft_consensus.cc:2781] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:17.088667 19966 raft_consensus.cc:686] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Becoming Leader. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Running, Role: LEADER
I20240502 07:58:17.088743 19955 ts_tablet_manager.cc:1398] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:17.088765 19966 consensus_queue.cc:237] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:17.089802 18938 catalog_manager.cc:5484] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } health_report { overall_health: HEALTHY } } }
W20240502 07:58:17.184352 20078 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:17.184537 20078 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:17.186971 20078 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_3'
INSERT report
rows total: 2000
time total: 65.3209 ms
time per row: 0.0326605 ms
I20240502 07:58:17.285413 18942 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:50010:
name: "rebalance_test_table_4"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:17.289765 19067 tablet_service.cc:1467] Processing CreateTablet for tablet 53f83cc7a58447158d0dae01fe31adba (DEFAULT_TABLE table=rebalance_test_table_4 [id=db78e3e5980d48a3886a4b5403848cb3]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:17.289887 19197 tablet_service.cc:1467] Processing CreateTablet for tablet 63baa0ba36f742ca91fddceb0e4b9617 (DEFAULT_TABLE table=rebalance_test_table_4 [id=db78e3e5980d48a3886a4b5403848cb3]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:17.289938 19066 tablet_service.cc:1467] Processing CreateTablet for tablet ca8eef98ed8247d9b5a7086ec2b0f56b (DEFAULT_TABLE table=rebalance_test_table_4 [id=db78e3e5980d48a3886a4b5403848cb3]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:17.290050 19067 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 53f83cc7a58447158d0dae01fe31adba. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:17.290186 19197 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 63baa0ba36f742ca91fddceb0e4b9617. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:17.290187 19066 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ca8eef98ed8247d9b5a7086ec2b0f56b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:17.291658 19955 tablet_bootstrap.cc:492] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118: Bootstrap starting.
I20240502 07:58:17.291985 19956 tablet_bootstrap.cc:492] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99: Bootstrap starting.
I20240502 07:58:17.292385 19955 tablet_bootstrap.cc:654] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:17.292749 19956 tablet_bootstrap.cc:654] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:17.293294 19955 tablet_bootstrap.cc:492] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118: No bootstrap required, opened a new log
I20240502 07:58:17.293360 19955 ts_tablet_manager.cc:1367] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:17.293689 19956 tablet_bootstrap.cc:492] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99: No bootstrap required, opened a new log
I20240502 07:58:17.293752 19956 ts_tablet_manager.cc:1367] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:17.293758 19955 raft_consensus.cc:348] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:17.293843 19955 raft_consensus.cc:374] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:17.293869 19955 raft_consensus.cc:729] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Initialized, Role: FOLLOWER
I20240502 07:58:17.293969 19955 consensus_queue.cc:260] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:17.294028 19955 raft_consensus.cc:388] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:17.294055 19955 raft_consensus.cc:482] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:17.294088 19955 raft_consensus.cc:3037] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:17.294126 19956 raft_consensus.cc:348] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:17.294199 19956 raft_consensus.cc:374] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:17.294224 19956 raft_consensus.cc:729] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Initialized, Role: FOLLOWER
I20240502 07:58:17.294312 19956 consensus_queue.cc:260] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:17.294364 19956 raft_consensus.cc:388] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:17.294390 19956 raft_consensus.cc:482] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:17.294421 19956 raft_consensus.cc:3037] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:17.294826 19955 raft_consensus.cc:504] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:17.294953 19955 leader_election.cc:302] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 3a68f86d2b9d44108fbb383fd0ec3118; no voters:
I20240502 07:58:17.295033 19955 leader_election.cc:288] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:17.295084 19960 raft_consensus.cc:2781] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:17.295167 19960 raft_consensus.cc:686] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Becoming Leader. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Running, Role: LEADER
I20240502 07:58:17.295266 19955 ts_tablet_manager.cc:1398] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 07:58:17.295261 19960 consensus_queue.cc:237] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } }
I20240502 07:58:17.295152 19956 raft_consensus.cc:504] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:17.295544 19956 leader_election.cc:302] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dfc05317d0c04450af55ceabe36f6a99; no voters:
I20240502 07:58:17.295643 19956 leader_election.cc:288] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:17.295785 19959 raft_consensus.cc:2781] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:17.295863 19959 raft_consensus.cc:686] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Becoming Leader. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Running, Role: LEADER
I20240502 07:58:17.295877 19956 ts_tablet_manager.cc:1398] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:17.295989 19956 tablet_bootstrap.cc:492] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99: Bootstrap starting.
I20240502 07:58:17.295953 19959 consensus_queue.cc:237] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:17.296869 19956 tablet_bootstrap.cc:654] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:17.296921 18942 catalog_manager.cc:5484] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:17.297360 18943 catalog_manager.cc:5484] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: term changed from 0 to 1, leader changed from <none> to dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:17.298561 19956 tablet_bootstrap.cc:492] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99: No bootstrap required, opened a new log
I20240502 07:58:17.298625 19956 ts_tablet_manager.cc:1367] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:17.299180 19956 raft_consensus.cc:348] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:17.299248 19956 raft_consensus.cc:374] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:17.299276 19956 raft_consensus.cc:729] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Initialized, Role: FOLLOWER
I20240502 07:58:17.299376 19956 consensus_queue.cc:260] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:17.299438 19956 raft_consensus.cc:388] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:17.299472 19956 raft_consensus.cc:482] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:17.299537 19956 raft_consensus.cc:3037] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:17.300098 19956 raft_consensus.cc:504] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:17.300204 19956 leader_election.cc:302] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: dfc05317d0c04450af55ceabe36f6a99; no voters:
I20240502 07:58:17.300287 19956 leader_election.cc:288] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:17.300348 19959 raft_consensus.cc:2781] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:17.300436 19959 raft_consensus.cc:686] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Becoming Leader. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Running, Role: LEADER
I20240502 07:58:17.300499 19956 ts_tablet_manager.cc:1398] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:17.300570 19959 consensus_queue.cc:237] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } }
I20240502 07:58:17.301702 18942 catalog_manager.cc:5484] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: term changed from 0 to 1, leader changed from <none> to dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } health_report { overall_health: HEALTHY } } }
W20240502 07:58:17.394984 20110 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:17.395144 20110 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:17.397652 20110 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_4'
INSERT report
rows total: 2000
time total: 62.8886 ms
time per row: 0.0314443 ms
I20240502 07:58:17.493762 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.131:42907
--local_ip_for_outbound_sockets=127.16.85.131
--tserver_master_addrs=127.16.85.190:41469
--webserver_port=39243
--webserver_interface=127.16.85.131
--builtin_ntp_servers=127.16.85.148:33323
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 07:58:17.586444 20136 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:17.586588 20136 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:17.586623 20136 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:17.589260 20136 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:17.589308 20136 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:17.589363 20136 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.131
I20240502 07:58:17.592499 20136 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33323
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.131:42907
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.16.85.131
--webserver_port=39243
--tserver_master_addrs=127.16.85.190:41469
--never_fsync=true
--heap_profile_path=/tmp/kudu.20136
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.131
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:17.593140 20136 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:17.594012 20136 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:17.600798 20141 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:17.601377 20142 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:17.602155 20144 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:17.602929 20136 server_base.cc:1034] running on GCE node
I20240502 07:58:17.603219 20136 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:17.603624 20136 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:17.604709 20136 hybrid_clock.cc:648] HybridClock initialized: now 1714636697604698 us; error 20 us; skew 500 ppm
I20240502 07:58:17.606098 20136 webserver.cc:460] Webserver started at http://127.16.85.131:39243/ using document root <none> and password file <none>
I20240502 07:58:17.606516 20136 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:17.606575 20136 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:17.609251 20136 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:17.610855 20150 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:17.611740 20136 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:17.611827 20136 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "cbc178a0a2f545658c5961645415eb25"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:17.612141 20136 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:17.623135 20136 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:17.623555 20136 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:17.623675 20136 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:17.624086 20136 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:17.624785 20136 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:17.624828 20136 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:17.624879 20136 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:17.624907 20136 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:17.633347 20136 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.131:42907
I20240502 07:58:17.633387 20263 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.131:42907 every 8 connection(s)
I20240502 07:58:17.634152 20136 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 07:58:17.638788 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 20136
I20240502 07:58:17.639171 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.132:44867
--local_ip_for_outbound_sockets=127.16.85.132
--tserver_master_addrs=127.16.85.190:41469
--webserver_port=37199
--webserver_interface=127.16.85.132
--builtin_ntp_servers=127.16.85.148:33323
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:17.642135 20264 heartbeater.cc:347] Connected to a master server at 127.16.85.190:41469
I20240502 07:58:17.642316 20264 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:17.642828 20264 heartbeater.cc:511] Master 127.16.85.190:41469 requested a full tablet report, sending...
I20240502 07:58:17.643777 18943 ts_manager.cc:194] Re-registered known tserver with Master: cbc178a0a2f545658c5961645415eb25 (127.16.85.131:42907)
I20240502 07:58:17.644260 18943 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.131:38237
W20240502 07:58:17.731091 20267 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:17.731273 20267 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:17.731346 20267 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:17.733911 20267 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:17.733965 20267 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:17.734041 20267 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.132
I20240502 07:58:17.736902 20267 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33323
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.132:44867
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.16.85.132
--webserver_port=37199
--tserver_master_addrs=127.16.85.190:41469
--never_fsync=true
--heap_profile_path=/tmp/kudu.20267
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.132
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:17.737496 20267 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:17.738320 20267 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:17.745038 20272 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:17.745525 20275 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:17.746176 20273 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:17.747030 20267 server_base.cc:1034] running on GCE node
I20240502 07:58:17.747295 20267 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:17.747684 20267 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:17.748828 20267 hybrid_clock.cc:648] HybridClock initialized: now 1714636697748807 us; error 31 us; skew 500 ppm
I20240502 07:58:17.750284 20267 webserver.cc:460] Webserver started at http://127.16.85.132:37199/ using document root <none> and password file <none>
I20240502 07:58:17.750715 20267 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:17.750774 20267 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:17.753412 20267 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.000s sys 0.003s
I20240502 07:58:17.754829 20281 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:17.755613 20267 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:17.755693 20267 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "278943ee039c48988108fc425b366ff7"
format_stamp: "Formatted at 2024-05-02 07:58:11 on dist-test-slave-7df7"
I20240502 07:58:17.756007 20267 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:17.765724 20267 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:17.766146 20267 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:17.766264 20267 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:17.766642 20267 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:17.767287 20267 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:17.767324 20267 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:17.767359 20267 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:17.767385 20267 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:17.774978 20267 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.132:44867
I20240502 07:58:17.774999 20394 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.132:44867 every 8 connection(s)
I20240502 07:58:17.775899 20267 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 07:58:17.782617 20395 heartbeater.cc:347] Connected to a master server at 127.16.85.190:41469
I20240502 07:58:17.782774 20395 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:17.783227 20395 heartbeater.cc:511] Master 127.16.85.190:41469 requested a full tablet report, sending...
I20240502 07:58:17.784005 18943 ts_manager.cc:194] Re-registered known tserver with Master: 278943ee039c48988108fc425b366ff7 (127.16.85.132:44867)
I20240502 07:58:17.784490 18943 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.132:54377
I20240502 07:58:17.784636 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 20267
I20240502 07:58:17.785023 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.133:36251
--local_ip_for_outbound_sockets=127.16.85.133
--tserver_master_addrs=127.16.85.190:41469
--webserver_port=33717
--webserver_interface=127.16.85.133
--builtin_ntp_servers=127.16.85.148:33323
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 07:58:17.878258 20398 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:17.878410 20398 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:17.878443 20398 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:17.881033 20398 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:17.881083 20398 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:17.881145 20398 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.133
I20240502 07:58:17.883996 20398 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33323
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.133:36251
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.16.85.133
--webserver_port=33717
--tserver_master_addrs=127.16.85.190:41469
--never_fsync=true
--heap_profile_path=/tmp/kudu.20398
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.133
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:17.884577 20398 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:17.885469 20398 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:17.892814 20406 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:17.893569 20404 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:17.894125 20403 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:17.894731 20398 server_base.cc:1034] running on GCE node
I20240502 07:58:17.895036 20398 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:17.895426 20398 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:17.896550 20398 hybrid_clock.cc:648] HybridClock initialized: now 1714636697896536 us; error 27 us; skew 500 ppm
I20240502 07:58:17.898043 20398 webserver.cc:460] Webserver started at http://127.16.85.133:33717/ using document root <none> and password file <none>
I20240502 07:58:17.898487 20398 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:17.898550 20398 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:17.901399 20398 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.003s sys 0.000s
I20240502 07:58:17.902912 20412 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:17.903703 20398 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:17.903788 20398 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "3a4ce39a207341a4883e10e934b839b4"
format_stamp: "Formatted at 2024-05-02 07:58:12 on dist-test-slave-7df7"
I20240502 07:58:17.904140 20398 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:17.917302 20398 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:17.917735 20398 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:17.917865 20398 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:17.918277 20398 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:17.918979 20398 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:17.919020 20398 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:17.919065 20398 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:17.919093 20398 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:17.926301 20398 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.133:36251
I20240502 07:58:17.926326 20525 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.133:36251 every 8 connection(s)
I20240502 07:58:17.927063 20398 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 07:58:17.929589 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 20398
I20240502 07:58:17.929917 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.134:36697
--local_ip_for_outbound_sockets=127.16.85.134
--tserver_master_addrs=127.16.85.190:41469
--webserver_port=46783
--webserver_interface=127.16.85.134
--builtin_ntp_servers=127.16.85.148:33323
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:17.934597 20526 heartbeater.cc:347] Connected to a master server at 127.16.85.190:41469
I20240502 07:58:17.934796 20526 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:17.935209 20526 heartbeater.cc:511] Master 127.16.85.190:41469 requested a full tablet report, sending...
I20240502 07:58:17.936056 18943 ts_manager.cc:194] Re-registered known tserver with Master: 3a4ce39a207341a4883e10e934b839b4 (127.16.85.133:36251)
I20240502 07:58:17.936528 18943 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.133:52617
W20240502 07:58:18.026875 20529 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:18.027050 20529 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:18.027117 20529 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:18.029856 20529 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:18.029923 20529 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:18.030004 20529 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.134
I20240502 07:58:18.033282 20529 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33323
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.134:36697
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.16.85.134
--webserver_port=46783
--tserver_master_addrs=127.16.85.190:41469
--never_fsync=true
--heap_profile_path=/tmp/kudu.20529
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.134
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:18.034027 20529 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:18.035014 20529 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:18.042673 20535 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:18.043200 20534 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:18.044023 20537 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:18.044821 20529 server_base.cc:1034] running on GCE node
I20240502 07:58:18.045125 20529 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:18.045598 20529 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:18.046772 20529 hybrid_clock.cc:648] HybridClock initialized: now 1714636698046764 us; error 21 us; skew 500 ppm
I20240502 07:58:18.048331 20529 webserver.cc:460] Webserver started at http://127.16.85.134:46783/ using document root <none> and password file <none>
I20240502 07:58:18.048827 20529 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:18.048908 20529 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:18.051916 20529 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:18.053459 20543 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:18.054327 20529 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:18.054425 20529 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "74506b6a243b450398cf31bdabf3eccf"
format_stamp: "Formatted at 2024-05-02 07:58:12 on dist-test-slave-7df7"
I20240502 07:58:18.054769 20529 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:18.087415 20529 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:18.087960 20529 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:18.088079 20529 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:18.088511 20529 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:18.089344 20529 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:18.089388 20529 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:18.089424 20529 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:18.089443 20529 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:18.097072 20529 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.134:36697
I20240502 07:58:18.097096 20656 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.134:36697 every 8 connection(s)
I20240502 07:58:18.097966 20529 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 07:58:18.105137 20657 heartbeater.cc:347] Connected to a master server at 127.16.85.190:41469
I20240502 07:58:18.105214 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 20529
I20240502 07:58:18.105340 20657 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:18.105588 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.135:45589
--local_ip_for_outbound_sockets=127.16.85.135
--tserver_master_addrs=127.16.85.190:41469
--webserver_port=45259
--webserver_interface=127.16.85.135
--builtin_ntp_servers=127.16.85.148:33323
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:18.105860 20657 heartbeater.cc:511] Master 127.16.85.190:41469 requested a full tablet report, sending...
I20240502 07:58:18.106844 18943 ts_manager.cc:194] Re-registered known tserver with Master: 74506b6a243b450398cf31bdabf3eccf (127.16.85.134:36697)
I20240502 07:58:18.107617 18943 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.134:45543
W20240502 07:58:18.201905 20660 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:18.202080 20660 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:18.202116 20660 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:18.204557 20660 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:18.204599 20660 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:18.204682 20660 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.135
I20240502 07:58:18.207617 20660 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33323
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.135:45589
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.16.85.135
--webserver_port=45259
--tserver_master_addrs=127.16.85.190:41469
--never_fsync=true
--heap_profile_path=/tmp/kudu.20660
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.135
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:18.208369 20660 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:18.209395 20660 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:18.217392 20668 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:18.218052 20666 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:18.218659 20665 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:18.219300 20660 server_base.cc:1034] running on GCE node
I20240502 07:58:18.219599 20660 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:18.220093 20660 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:18.221256 20660 hybrid_clock.cc:648] HybridClock initialized: now 1714636698221236 us; error 25 us; skew 500 ppm
I20240502 07:58:18.222925 20660 webserver.cc:460] Webserver started at http://127.16.85.135:45259/ using document root <none> and password file <none>
I20240502 07:58:18.223481 20660 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:18.223572 20660 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:18.226475 20660 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.003s sys 0.000s
I20240502 07:58:18.228080 20674 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:18.228937 20660 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:18.229030 20660 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "f81ce2156e854e0ab96f4b6fb2077714"
format_stamp: "Formatted at 2024-05-02 07:58:12 on dist-test-slave-7df7"
I20240502 07:58:18.229389 20660 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:18.252115 20660 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:18.252697 20660 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:18.252832 20660 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:18.253304 20660 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:18.254257 20681 ts_tablet_manager.cc:516] Loading tablet metadata (0/1 complete)
I20240502 07:58:18.257053 20660 ts_tablet_manager.cc:559] Loaded tablet metadata (1 total tablets, 1 live tablets)
I20240502 07:58:18.257123 20660 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.003s user 0.000s sys 0.000s
I20240502 07:58:18.257175 20660 ts_tablet_manager.cc:574] Registering tablets (0/1 complete)
I20240502 07:58:18.258793 20660 ts_tablet_manager.cc:590] Registered 1 tablets
I20240502 07:58:18.258837 20660 ts_tablet_manager.cc:569] Time spent register tablets: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:18.259056 20681 tablet_bootstrap.cc:492] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714: Bootstrap starting.
I20240502 07:58:18.267650 20660 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.135:45589
I20240502 07:58:18.268631 20660 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 07:58:18.269742 20788 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.135:45589 every 8 connection(s)
I20240502 07:58:18.271678 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 20660
I20240502 07:58:18.272538 20681 log.cc:824] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:18.274806 20681 tablet_bootstrap.cc:492] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714: Bootstrap replayed 1/1 log segments. Stats: ops{read=1 overwritten=0 applied=1 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:18.275341 20681 tablet_bootstrap.cc:492] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714: Bootstrap complete.
I20240502 07:58:18.276352 20681 ts_tablet_manager.cc:1367] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714: Time spent bootstrapping tablet: real 0.017s user 0.013s sys 0.004s
I20240502 07:58:18.278157 20789 heartbeater.cc:347] Connected to a master server at 127.16.85.190:41469
I20240502 07:58:18.278357 20789 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:18.278874 20789 heartbeater.cc:511] Master 127.16.85.190:41469 requested a full tablet report, sending...
I20240502 07:58:18.279186 20681 raft_consensus.cc:348] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } }
I20240502 07:58:18.279525 20681 raft_consensus.cc:729] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: f81ce2156e854e0ab96f4b6fb2077714, State: Initialized, Role: FOLLOWER
I20240502 07:58:18.279937 20681 consensus_queue.cc:260] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 1, Last appended: 1.1, Last appended by leader: 1, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } }
I20240502 07:58:18.280051 20681 raft_consensus.cc:388] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 1 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:18.280125 20681 raft_consensus.cc:482] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 1 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:18.280216 20681 raft_consensus.cc:3037] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:18.280575 18943 ts_manager.cc:194] Re-registered known tserver with Master: f81ce2156e854e0ab96f4b6fb2077714 (127.16.85.135:45589)
I20240502 07:58:18.281555 18943 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.135:43289
I20240502 07:58:18.281996 20681 raft_consensus.cc:504] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } }
I20240502 07:58:18.282279 20681 leader_election.cc:302] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: f81ce2156e854e0ab96f4b6fb2077714; no voters:
I20240502 07:58:18.282564 20681 leader_election.cc:288] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [CANDIDATE]: Term 2 election: Requested vote from peers
I20240502 07:58:18.282708 20794 raft_consensus.cc:2781] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:18.282959 20794 raft_consensus.cc:686] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [term 2 LEADER]: Becoming Leader. State: Replica: f81ce2156e854e0ab96f4b6fb2077714, State: Running, Role: LEADER
I20240502 07:58:18.283272 20681 ts_tablet_manager.cc:1398] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714: Time spent starting tablet: real 0.007s user 0.007s sys 0.000s
I20240502 07:58:18.283286 20794 consensus_queue.cc:237] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 1, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } }
I20240502 07:58:18.283579 20789 heartbeater.cc:503] Master 127.16.85.190:41469 was elected leader, sending a full tablet report...
I20240502 07:58:18.285188 18943 catalog_manager.cc:5484] T 3b5a6f09d531470cbca30aa50654dcbf P f81ce2156e854e0ab96f4b6fb2077714 reported cstate change: term changed from 1 to 2. New cstate: current_term: 2 leader_uuid: "f81ce2156e854e0ab96f4b6fb2077714" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:18.443070 20198 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:18.443847 20460 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:18.444223 20329 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:18.448228 20591 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:18.448945 20713 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:18.449466 19066 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 07:58:18.461634 19197 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 07:58:18.565798 20329 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:18.566309 20460 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:18.571238 19066 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 07:58:18.571991 20591 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:18.572852 20713 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:18.573479 20198 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:18.586347 19197 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 07:58:18.617512 19087 consensus_queue.cc:237] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: true } }
I20240502 07:58:18.618714 20888 raft_consensus.cc:2932] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER cbc178a0a2f545658c5961645415eb25 (127.16.85.131) added. New config: { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: true } } }
I20240502 07:58:18.621359 18940 catalog_manager.cc:5484] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: config changed from index -1 to 4, NON_VOTER cbc178a0a2f545658c5961645415eb25 (127.16.85.131) added. New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:18.624156 19022 consensus_peers.cc:485] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 -> Peer cbc178a0a2f545658c5961645415eb25 (127.16.85.131:42907): Couldn't send request to peer cbc178a0a2f545658c5961645415eb25. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 53f83cc7a58447158d0dae01fe31adba. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:18.627439 19218 consensus_queue.cc:237] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: true } }
I20240502 07:58:18.628505 20898 raft_consensus.cc:2932] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74506b6a243b450398cf31bdabf3eccf (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: true } } }
I20240502 07:58:18.630954 18936 catalog_manager.cc:5484] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 reported cstate change: config changed from index -1 to 4, NON_VOTER 74506b6a243b450398cf31bdabf3eccf (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:18.636847 19087 consensus_queue.cc:237] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: true } }
W20240502 07:58:18.636982 19154 consensus_peers.cc:485] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 -> Peer 74506b6a243b450398cf31bdabf3eccf (127.16.85.134:36697): Couldn't send request to peer 74506b6a243b450398cf31bdabf3eccf. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0cb1145d5f6343a1b205bb59221e95f4. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:18.637643 20888 raft_consensus.cc:2932] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 278943ee039c48988108fc425b366ff7 (127.16.85.132) added. New config: { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: true } } }
I20240502 07:58:18.639616 18936 catalog_manager.cc:5484] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: config changed from index -1 to 4, NON_VOTER 278943ee039c48988108fc425b366ff7 (127.16.85.132) added. New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:18.641955 19021 consensus_peers.cc:485] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 -> Peer 278943ee039c48988108fc425b366ff7 (127.16.85.132:44867): Couldn't send request to peer 278943ee039c48988108fc425b366ff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 12852f6b0d7242208c3f2beea408bc89. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:18.645401 19218 consensus_queue.cc:237] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: true } }
I20240502 07:58:18.646065 20897 raft_consensus.cc:2932] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3a4ce39a207341a4883e10e934b839b4 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: true } } }
I20240502 07:58:18.646203 20264 heartbeater.cc:503] Master 127.16.85.190:41469 was elected leader, sending a full tablet report...
I20240502 07:58:18.647872 18936 catalog_manager.cc:5484] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 reported cstate change: config changed from index -1 to 4, NON_VOTER 3a4ce39a207341a4883e10e934b839b4 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:18.650856 19154 consensus_peers.cc:485] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 -> Peer 3a4ce39a207341a4883e10e934b839b4 (127.16.85.133:36251): Couldn't send request to peer 3a4ce39a207341a4883e10e934b839b4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 63baa0ba36f742ca91fddceb0e4b9617. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:18.653741 19087 consensus_queue.cc:237] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: true } }
I20240502 07:58:18.654435 20889 raft_consensus.cc:2932] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER f81ce2156e854e0ab96f4b6fb2077714 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: true } } }
I20240502 07:58:18.656886 18936 catalog_manager.cc:5484] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: config changed from index -1 to 4, NON_VOTER f81ce2156e854e0ab96f4b6fb2077714 (127.16.85.135) added. New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:18.657650 19022 consensus_peers.cc:485] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 -> Peer f81ce2156e854e0ab96f4b6fb2077714 (127.16.85.135:45589): Couldn't send request to peer f81ce2156e854e0ab96f4b6fb2077714. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 708e377a224e40deab16180eb8f32c82. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:18.662051 19087 consensus_queue.cc:237] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: true } }
I20240502 07:58:18.662660 20889 raft_consensus.cc:2932] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER cbc178a0a2f545658c5961645415eb25 (127.16.85.131) added. New config: { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: true } } }
W20240502 07:58:18.663014 19022 consensus_peers.cc:485] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 -> Peer cbc178a0a2f545658c5961645415eb25 (127.16.85.131:42907): Couldn't send request to peer cbc178a0a2f545658c5961645415eb25. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f8550e00f0ba4b5ea52822f99add2cfe. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:18.664618 18940 catalog_manager.cc:5484] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: config changed from index -1 to 4, NON_VOTER cbc178a0a2f545658c5961645415eb25 (127.16.85.131) added. New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:18.669044 19218 consensus_queue.cc:237] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: true } }
I20240502 07:58:18.669699 20898 raft_consensus.cc:2932] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74506b6a243b450398cf31bdabf3eccf (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: true } } }
W20240502 07:58:18.670096 19154 consensus_peers.cc:485] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 -> Peer 74506b6a243b450398cf31bdabf3eccf (127.16.85.134:36697): Couldn't send request to peer 74506b6a243b450398cf31bdabf3eccf. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 95b1058e6dd5473ca41d534137ebb3eb. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:18.671823 18936 catalog_manager.cc:5484] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 reported cstate change: config changed from index -1 to 4, NON_VOTER 74506b6a243b450398cf31bdabf3eccf (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:18.676398 19218 consensus_queue.cc:237] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: true } }
I20240502 07:58:18.677079 20930 raft_consensus.cc:2932] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 278943ee039c48988108fc425b366ff7 (127.16.85.132) added. New config: { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: true } } }
I20240502 07:58:18.678705 18936 catalog_manager.cc:5484] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 reported cstate change: config changed from index -1 to 4, NON_VOTER 278943ee039c48988108fc425b366ff7 (127.16.85.132) added. New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:18.681289 19152 consensus_peers.cc:485] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 -> Peer 278943ee039c48988108fc425b366ff7 (127.16.85.132:44867): Couldn't send request to peer 278943ee039c48988108fc425b366ff7. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: af6d015cb2154a29970c5a7a4116efe9. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:18.684160 19087 consensus_queue.cc:237] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: true } }
I20240502 07:58:18.684856 20889 raft_consensus.cc:2932] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3a4ce39a207341a4883e10e934b839b4 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: true } } }
I20240502 07:58:18.686609 18936 catalog_manager.cc:5484] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: config changed from index -1 to 4, NON_VOTER 3a4ce39a207341a4883e10e934b839b4 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:18.687878 19023 consensus_peers.cc:485] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 -> Peer 3a4ce39a207341a4883e10e934b839b4 (127.16.85.133:36251): Couldn't send request to peer 3a4ce39a207341a4883e10e934b839b4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 823e3d11a041417e90344de4350fee72. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:18.786345 20395 heartbeater.cc:503] Master 127.16.85.190:41469 was elected leader, sending a full tablet report...
I20240502 07:58:18.938107 20526 heartbeater.cc:503] Master 127.16.85.190:41469 was elected leader, sending a full tablet report...
I20240502 07:58:19.017233 21033 ts_tablet_manager.cc:902] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf: Initiating tablet copy from peer 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130:33077)
I20240502 07:58:19.017807 21033 tablet_copy_client.cc:308] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33077
I20240502 07:58:19.020871 19238 tablet_copy_service.cc:140] P 3a68f86d2b9d44108fbb383fd0ec3118: Received BeginTabletCopySession request for tablet 0cb1145d5f6343a1b205bb59221e95f4 from peer 74506b6a243b450398cf31bdabf3eccf ({username='slave'} at 127.16.85.134:43449)
I20240502 07:58:19.020992 19238 tablet_copy_service.cc:161] P 3a68f86d2b9d44108fbb383fd0ec3118: Beginning new tablet copy session on tablet 0cb1145d5f6343a1b205bb59221e95f4 from peer 74506b6a243b450398cf31bdabf3eccf at {username='slave'} at 127.16.85.134:43449: session id = 74506b6a243b450398cf31bdabf3eccf-0cb1145d5f6343a1b205bb59221e95f4
I20240502 07:58:19.021999 19238 tablet_copy_source_session.cc:205] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:19.024642 21033 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0cb1145d5f6343a1b205bb59221e95f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:19.028910 21033 tablet_copy_client.cc:783] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:19.029057 21033 tablet_copy_client.cc:647] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:19.030372 21033 tablet_copy_client.cc:523] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:19.031639 21033 tablet_bootstrap.cc:492] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf: Bootstrap starting.
I20240502 07:58:19.035832 21033 log.cc:824] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:19.052052 21033 tablet_bootstrap.cc:492] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:19.052285 21033 tablet_bootstrap.cc:492] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf: Bootstrap complete.
I20240502 07:58:19.052480 21033 ts_tablet_manager.cc:1367] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf: Time spent bootstrapping tablet: real 0.021s user 0.017s sys 0.001s
I20240502 07:58:19.054286 21033 raft_consensus.cc:348] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: true } }
I20240502 07:58:19.054445 21033 raft_consensus.cc:729] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 74506b6a243b450398cf31bdabf3eccf, State: Initialized, Role: LEARNER
I20240502 07:58:19.054701 21033 consensus_queue.cc:260] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: true } }
I20240502 07:58:19.055181 20657 heartbeater.cc:503] Master 127.16.85.190:41469 was elected leader, sending a full tablet report...
I20240502 07:58:19.055338 21033 ts_tablet_manager.cc:1398] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf: Time spent starting tablet: real 0.003s user 0.006s sys 0.000s
I20240502 07:58:19.055905 19238 tablet_copy_service.cc:342] P 3a68f86d2b9d44108fbb383fd0ec3118: Request end of tablet copy session 74506b6a243b450398cf31bdabf3eccf-0cb1145d5f6343a1b205bb59221e95f4 received from {username='slave'} at 127.16.85.134:43449
I20240502 07:58:19.055974 19238 tablet_copy_service.cc:433] P 3a68f86d2b9d44108fbb383fd0ec3118: ending tablet copy session 74506b6a243b450398cf31bdabf3eccf-0cb1145d5f6343a1b205bb59221e95f4 on tablet 0cb1145d5f6343a1b205bb59221e95f4 with peer 74506b6a243b450398cf31bdabf3eccf
I20240502 07:58:19.088014 21041 ts_tablet_manager.cc:902] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25: Initiating tablet copy from peer dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153)
I20240502 07:58:19.088622 21041 tablet_copy_client.cc:308] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:44153
I20240502 07:58:19.088721 21045 ts_tablet_manager.cc:902] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7: Initiating tablet copy from peer 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130:33077)
I20240502 07:58:19.089481 21045 tablet_copy_client.cc:308] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33077
I20240502 07:58:19.092159 19107 tablet_copy_service.cc:140] P dfc05317d0c04450af55ceabe36f6a99: Received BeginTabletCopySession request for tablet f8550e00f0ba4b5ea52822f99add2cfe from peer cbc178a0a2f545658c5961645415eb25 ({username='slave'} at 127.16.85.131:50699)
I20240502 07:58:19.092298 19107 tablet_copy_service.cc:161] P dfc05317d0c04450af55ceabe36f6a99: Beginning new tablet copy session on tablet f8550e00f0ba4b5ea52822f99add2cfe from peer cbc178a0a2f545658c5961645415eb25 at {username='slave'} at 127.16.85.131:50699: session id = cbc178a0a2f545658c5961645415eb25-f8550e00f0ba4b5ea52822f99add2cfe
I20240502 07:58:19.092854 19238 tablet_copy_service.cc:140] P 3a68f86d2b9d44108fbb383fd0ec3118: Received BeginTabletCopySession request for tablet af6d015cb2154a29970c5a7a4116efe9 from peer 278943ee039c48988108fc425b366ff7 ({username='slave'} at 127.16.85.132:37811)
I20240502 07:58:19.092927 19238 tablet_copy_service.cc:161] P 3a68f86d2b9d44108fbb383fd0ec3118: Beginning new tablet copy session on tablet af6d015cb2154a29970c5a7a4116efe9 from peer 278943ee039c48988108fc425b366ff7 at {username='slave'} at 127.16.85.132:37811: session id = 278943ee039c48988108fc425b366ff7-af6d015cb2154a29970c5a7a4116efe9
I20240502 07:58:19.093432 19107 tablet_copy_source_session.cc:205] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:19.093569 19238 tablet_copy_source_session.cc:205] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:19.096469 21045 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet af6d015cb2154a29970c5a7a4116efe9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:19.096469 21041 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f8550e00f0ba4b5ea52822f99add2cfe. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:19.102116 21045 tablet_copy_client.cc:783] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:19.102274 21045 tablet_copy_client.cc:647] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:19.102558 21041 tablet_copy_client.cc:783] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:19.102707 21041 tablet_copy_client.cc:647] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:19.103595 21045 tablet_copy_client.cc:523] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:19.104126 21041 tablet_copy_client.cc:523] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:19.105203 21045 tablet_bootstrap.cc:492] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7: Bootstrap starting.
I20240502 07:58:19.105324 21053 ts_tablet_manager.cc:902] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25: Initiating tablet copy from peer dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153)
I20240502 07:58:19.105656 21041 tablet_bootstrap.cc:492] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25: Bootstrap starting.
I20240502 07:58:19.105787 21053 tablet_copy_client.cc:308] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:44153
I20240502 07:58:19.106168 19107 tablet_copy_service.cc:140] P dfc05317d0c04450af55ceabe36f6a99: Received BeginTabletCopySession request for tablet 53f83cc7a58447158d0dae01fe31adba from peer cbc178a0a2f545658c5961645415eb25 ({username='slave'} at 127.16.85.131:50699)
I20240502 07:58:19.106258 19107 tablet_copy_service.cc:161] P dfc05317d0c04450af55ceabe36f6a99: Beginning new tablet copy session on tablet 53f83cc7a58447158d0dae01fe31adba from peer cbc178a0a2f545658c5961645415eb25 at {username='slave'} at 127.16.85.131:50699: session id = cbc178a0a2f545658c5961645415eb25-53f83cc7a58447158d0dae01fe31adba
I20240502 07:58:19.106966 19107 tablet_copy_source_session.cc:205] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:19.107669 21053 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 53f83cc7a58447158d0dae01fe31adba. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:19.109164 21053 tablet_copy_client.cc:783] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:19.109325 21053 tablet_copy_client.cc:647] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:19.110237 21053 tablet_copy_client.cc:523] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:19.110877 21045 log.cc:824] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:19.110886 21041 log.cc:824] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:19.111071 21053 tablet_bootstrap.cc:492] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25: Bootstrap starting.
I20240502 07:58:19.129853 21053 tablet_bootstrap.cc:492] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:19.130100 21053 tablet_bootstrap.cc:492] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25: Bootstrap complete.
I20240502 07:58:19.130358 21053 ts_tablet_manager.cc:1367] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25: Time spent bootstrapping tablet: real 0.019s user 0.018s sys 0.000s
I20240502 07:58:19.132417 21053 raft_consensus.cc:348] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: true } }
I20240502 07:58:19.132627 21053 raft_consensus.cc:729] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: cbc178a0a2f545658c5961645415eb25, State: Initialized, Role: LEARNER
I20240502 07:58:19.132910 21053 consensus_queue.cc:260] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: true } }
I20240502 07:58:19.133610 21053 ts_tablet_manager.cc:1398] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25: Time spent starting tablet: real 0.003s user 0.004s sys 0.000s
I20240502 07:58:19.134550 19107 tablet_copy_service.cc:342] P dfc05317d0c04450af55ceabe36f6a99: Request end of tablet copy session cbc178a0a2f545658c5961645415eb25-53f83cc7a58447158d0dae01fe31adba received from {username='slave'} at 127.16.85.131:50699
I20240502 07:58:19.134626 19107 tablet_copy_service.cc:433] P dfc05317d0c04450af55ceabe36f6a99: ending tablet copy session cbc178a0a2f545658c5961645415eb25-53f83cc7a58447158d0dae01fe31adba on tablet 53f83cc7a58447158d0dae01fe31adba with peer cbc178a0a2f545658c5961645415eb25
I20240502 07:58:19.134954 21041 tablet_bootstrap.cc:492] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:19.135166 21041 tablet_bootstrap.cc:492] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25: Bootstrap complete.
I20240502 07:58:19.135296 21041 ts_tablet_manager.cc:1367] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25: Time spent bootstrapping tablet: real 0.030s user 0.025s sys 0.000s
I20240502 07:58:19.136246 21041 raft_consensus.cc:348] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: true } }
I20240502 07:58:19.136346 21041 raft_consensus.cc:729] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: cbc178a0a2f545658c5961645415eb25, State: Initialized, Role: LEARNER
I20240502 07:58:19.136400 21045 tablet_bootstrap.cc:492] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:19.136437 21041 consensus_queue.cc:260] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: true } }
I20240502 07:58:19.136718 21045 tablet_bootstrap.cc:492] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7: Bootstrap complete.
I20240502 07:58:19.136714 21041 ts_tablet_manager.cc:1398] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25: Time spent starting tablet: real 0.001s user 0.003s sys 0.000s
I20240502 07:58:19.136956 21045 ts_tablet_manager.cc:1367] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7: Time spent bootstrapping tablet: real 0.032s user 0.030s sys 0.000s
I20240502 07:58:19.137112 19107 tablet_copy_service.cc:342] P dfc05317d0c04450af55ceabe36f6a99: Request end of tablet copy session cbc178a0a2f545658c5961645415eb25-f8550e00f0ba4b5ea52822f99add2cfe received from {username='slave'} at 127.16.85.131:50699
I20240502 07:58:19.137187 19107 tablet_copy_service.cc:433] P dfc05317d0c04450af55ceabe36f6a99: ending tablet copy session cbc178a0a2f545658c5961645415eb25-f8550e00f0ba4b5ea52822f99add2cfe on tablet f8550e00f0ba4b5ea52822f99add2cfe with peer cbc178a0a2f545658c5961645415eb25
I20240502 07:58:19.139101 21045 raft_consensus.cc:348] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: true } }
I20240502 07:58:19.139266 21045 raft_consensus.cc:729] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 278943ee039c48988108fc425b366ff7, State: Initialized, Role: LEARNER
I20240502 07:58:19.139528 21045 consensus_queue.cc:260] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: true } }
I20240502 07:58:19.140132 21045 ts_tablet_manager.cc:1398] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 07:58:19.140713 19238 tablet_copy_service.cc:342] P 3a68f86d2b9d44108fbb383fd0ec3118: Request end of tablet copy session 278943ee039c48988108fc425b366ff7-af6d015cb2154a29970c5a7a4116efe9 received from {username='slave'} at 127.16.85.132:37811
I20240502 07:58:19.140794 19238 tablet_copy_service.cc:433] P 3a68f86d2b9d44108fbb383fd0ec3118: ending tablet copy session 278943ee039c48988108fc425b366ff7-af6d015cb2154a29970c5a7a4116efe9 on tablet af6d015cb2154a29970c5a7a4116efe9 with peer 278943ee039c48988108fc425b366ff7
I20240502 07:58:19.141678 21063 ts_tablet_manager.cc:902] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4: Initiating tablet copy from peer 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130:33077)
I20240502 07:58:19.142293 21063 tablet_copy_client.cc:308] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33077
I20240502 07:58:19.145166 19238 tablet_copy_service.cc:140] P 3a68f86d2b9d44108fbb383fd0ec3118: Received BeginTabletCopySession request for tablet 63baa0ba36f742ca91fddceb0e4b9617 from peer 3a4ce39a207341a4883e10e934b839b4 ({username='slave'} at 127.16.85.133:41075)
I20240502 07:58:19.145258 19238 tablet_copy_service.cc:161] P 3a68f86d2b9d44108fbb383fd0ec3118: Beginning new tablet copy session on tablet 63baa0ba36f742ca91fddceb0e4b9617 from peer 3a4ce39a207341a4883e10e934b839b4 at {username='slave'} at 127.16.85.133:41075: session id = 3a4ce39a207341a4883e10e934b839b4-63baa0ba36f742ca91fddceb0e4b9617
I20240502 07:58:19.145852 19238 tablet_copy_source_session.cc:205] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:19.148061 21063 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 63baa0ba36f742ca91fddceb0e4b9617. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:19.152489 21063 tablet_copy_client.cc:783] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:19.152631 21063 tablet_copy_client.cc:647] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:19.153745 21063 tablet_copy_client.cc:523] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:19.154834 21063 tablet_bootstrap.cc:492] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4: Bootstrap starting.
I20240502 07:58:19.158457 21069 ts_tablet_manager.cc:902] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714: Initiating tablet copy from peer dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153)
I20240502 07:58:19.159075 21063 log.cc:824] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:19.159119 21069 tablet_copy_client.cc:308] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:44153
I20240502 07:58:19.162706 19107 tablet_copy_service.cc:140] P dfc05317d0c04450af55ceabe36f6a99: Received BeginTabletCopySession request for tablet 708e377a224e40deab16180eb8f32c82 from peer f81ce2156e854e0ab96f4b6fb2077714 ({username='slave'} at 127.16.85.135:52579)
I20240502 07:58:19.162788 19107 tablet_copy_service.cc:161] P dfc05317d0c04450af55ceabe36f6a99: Beginning new tablet copy session on tablet 708e377a224e40deab16180eb8f32c82 from peer f81ce2156e854e0ab96f4b6fb2077714 at {username='slave'} at 127.16.85.135:52579: session id = f81ce2156e854e0ab96f4b6fb2077714-708e377a224e40deab16180eb8f32c82
I20240502 07:58:19.163378 19107 tablet_copy_source_session.cc:205] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:19.164722 21069 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 708e377a224e40deab16180eb8f32c82. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:19.166905 21069 tablet_copy_client.cc:783] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:19.167043 21069 tablet_copy_client.cc:647] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:19.168162 21069 tablet_copy_client.cc:523] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:19.169198 21069 tablet_bootstrap.cc:492] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714: Bootstrap starting.
I20240502 07:58:19.179397 21063 tablet_bootstrap.cc:492] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:19.179654 21063 tablet_bootstrap.cc:492] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4: Bootstrap complete.
I20240502 07:58:19.179889 21063 ts_tablet_manager.cc:1367] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4: Time spent bootstrapping tablet: real 0.025s user 0.023s sys 0.000s
I20240502 07:58:19.182174 21063 raft_consensus.cc:348] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: true } }
I20240502 07:58:19.182379 21063 raft_consensus.cc:729] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3a4ce39a207341a4883e10e934b839b4, State: Initialized, Role: LEARNER
I20240502 07:58:19.182734 21063 consensus_queue.cc:260] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: true } }
I20240502 07:58:19.183518 21063 ts_tablet_manager.cc:1398] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4: Time spent starting tablet: real 0.004s user 0.006s sys 0.000s
I20240502 07:58:19.184247 19238 tablet_copy_service.cc:342] P 3a68f86d2b9d44108fbb383fd0ec3118: Request end of tablet copy session 3a4ce39a207341a4883e10e934b839b4-63baa0ba36f742ca91fddceb0e4b9617 received from {username='slave'} at 127.16.85.133:41075
I20240502 07:58:19.184303 19238 tablet_copy_service.cc:433] P 3a68f86d2b9d44108fbb383fd0ec3118: ending tablet copy session 3a4ce39a207341a4883e10e934b839b4-63baa0ba36f742ca91fddceb0e4b9617 on tablet 63baa0ba36f742ca91fddceb0e4b9617 with peer 3a4ce39a207341a4883e10e934b839b4
I20240502 07:58:19.193699 21069 tablet_bootstrap.cc:492] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:19.193912 21069 tablet_bootstrap.cc:492] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714: Bootstrap complete.
I20240502 07:58:19.194041 21069 ts_tablet_manager.cc:1367] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714: Time spent bootstrapping tablet: real 0.025s user 0.020s sys 0.003s
I20240502 07:58:19.194458 21069 raft_consensus.cc:348] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: true } }
I20240502 07:58:19.194579 21069 raft_consensus.cc:729] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: f81ce2156e854e0ab96f4b6fb2077714, State: Initialized, Role: LEARNER
I20240502 07:58:19.194672 21069 consensus_queue.cc:260] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: true } }
I20240502 07:58:19.194984 21069 ts_tablet_manager.cc:1398] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 07:58:19.195502 19107 tablet_copy_service.cc:342] P dfc05317d0c04450af55ceabe36f6a99: Request end of tablet copy session f81ce2156e854e0ab96f4b6fb2077714-708e377a224e40deab16180eb8f32c82 received from {username='slave'} at 127.16.85.135:52579
I20240502 07:58:19.195559 19107 tablet_copy_service.cc:433] P dfc05317d0c04450af55ceabe36f6a99: ending tablet copy session f81ce2156e854e0ab96f4b6fb2077714-708e377a224e40deab16180eb8f32c82 on tablet 708e377a224e40deab16180eb8f32c82 with peer f81ce2156e854e0ab96f4b6fb2077714
I20240502 07:58:19.211634 21063 ts_tablet_manager.cc:902] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4: Initiating tablet copy from peer dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153)
I20240502 07:58:19.211989 21063 tablet_copy_client.cc:308] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:44153
I20240502 07:58:19.215162 19107 tablet_copy_service.cc:140] P dfc05317d0c04450af55ceabe36f6a99: Received BeginTabletCopySession request for tablet 823e3d11a041417e90344de4350fee72 from peer 3a4ce39a207341a4883e10e934b839b4 ({username='slave'} at 127.16.85.133:37207)
I20240502 07:58:19.215250 19107 tablet_copy_service.cc:161] P dfc05317d0c04450af55ceabe36f6a99: Beginning new tablet copy session on tablet 823e3d11a041417e90344de4350fee72 from peer 3a4ce39a207341a4883e10e934b839b4 at {username='slave'} at 127.16.85.133:37207: session id = 3a4ce39a207341a4883e10e934b839b4-823e3d11a041417e90344de4350fee72
I20240502 07:58:19.215802 19107 tablet_copy_source_session.cc:205] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:19.216394 21063 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 823e3d11a041417e90344de4350fee72. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:19.217849 21063 tablet_copy_client.cc:783] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:19.217972 21063 tablet_copy_client.cc:647] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:19.218850 21063 tablet_copy_client.cc:523] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:19.219607 21063 tablet_bootstrap.cc:492] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4: Bootstrap starting.
I20240502 07:58:19.231935 21033 ts_tablet_manager.cc:902] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf: Initiating tablet copy from peer 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130:33077)
I20240502 07:58:19.232385 21033 tablet_copy_client.cc:308] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33077
I20240502 07:58:19.232847 19238 tablet_copy_service.cc:140] P 3a68f86d2b9d44108fbb383fd0ec3118: Received BeginTabletCopySession request for tablet 95b1058e6dd5473ca41d534137ebb3eb from peer 74506b6a243b450398cf31bdabf3eccf ({username='slave'} at 127.16.85.134:43449)
I20240502 07:58:19.232940 19238 tablet_copy_service.cc:161] P 3a68f86d2b9d44108fbb383fd0ec3118: Beginning new tablet copy session on tablet 95b1058e6dd5473ca41d534137ebb3eb from peer 74506b6a243b450398cf31bdabf3eccf at {username='slave'} at 127.16.85.134:43449: session id = 74506b6a243b450398cf31bdabf3eccf-95b1058e6dd5473ca41d534137ebb3eb
I20240502 07:58:19.233629 19238 tablet_copy_source_session.cc:205] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:19.234272 21033 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 95b1058e6dd5473ca41d534137ebb3eb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:19.234800 21063 tablet_bootstrap.cc:492] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:19.234985 21063 tablet_bootstrap.cc:492] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4: Bootstrap complete.
I20240502 07:58:19.235110 21063 ts_tablet_manager.cc:1367] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4: Time spent bootstrapping tablet: real 0.016s user 0.012s sys 0.002s
I20240502 07:58:19.235492 21063 raft_consensus.cc:348] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: true } }
I20240502 07:58:19.235574 21063 raft_consensus.cc:729] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3a4ce39a207341a4883e10e934b839b4, State: Initialized, Role: LEARNER
I20240502 07:58:19.235677 21063 consensus_queue.cc:260] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: true } }
I20240502 07:58:19.235780 21033 tablet_copy_client.cc:783] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:19.235904 21033 tablet_copy_client.cc:647] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:19.235991 21063 ts_tablet_manager.cc:1398] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4: Time spent starting tablet: real 0.001s user 0.003s sys 0.000s
I20240502 07:58:19.236387 19107 tablet_copy_service.cc:342] P dfc05317d0c04450af55ceabe36f6a99: Request end of tablet copy session 3a4ce39a207341a4883e10e934b839b4-823e3d11a041417e90344de4350fee72 received from {username='slave'} at 127.16.85.133:37207
I20240502 07:58:19.236460 19107 tablet_copy_service.cc:433] P dfc05317d0c04450af55ceabe36f6a99: ending tablet copy session 3a4ce39a207341a4883e10e934b839b4-823e3d11a041417e90344de4350fee72 on tablet 823e3d11a041417e90344de4350fee72 with peer 3a4ce39a207341a4883e10e934b839b4
I20240502 07:58:19.237003 21033 tablet_copy_client.cc:523] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:19.237843 21033 tablet_bootstrap.cc:492] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf: Bootstrap starting.
I20240502 07:58:19.251906 21033 tablet_bootstrap.cc:492] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:19.252092 21033 tablet_bootstrap.cc:492] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf: Bootstrap complete.
I20240502 07:58:19.252231 21033 ts_tablet_manager.cc:1367] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf: Time spent bootstrapping tablet: real 0.014s user 0.011s sys 0.000s
I20240502 07:58:19.252595 21033 raft_consensus.cc:348] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: true } }
I20240502 07:58:19.252697 21033 raft_consensus.cc:729] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 74506b6a243b450398cf31bdabf3eccf, State: Initialized, Role: LEARNER
I20240502 07:58:19.252787 21033 consensus_queue.cc:260] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: true } }
I20240502 07:58:19.253015 21033 ts_tablet_manager.cc:1398] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf: Time spent starting tablet: real 0.001s user 0.004s sys 0.000s
I20240502 07:58:19.253407 19238 tablet_copy_service.cc:342] P 3a68f86d2b9d44108fbb383fd0ec3118: Request end of tablet copy session 74506b6a243b450398cf31bdabf3eccf-95b1058e6dd5473ca41d534137ebb3eb received from {username='slave'} at 127.16.85.134:43449
I20240502 07:58:19.253468 19238 tablet_copy_service.cc:433] P 3a68f86d2b9d44108fbb383fd0ec3118: ending tablet copy session 74506b6a243b450398cf31bdabf3eccf-95b1058e6dd5473ca41d534137ebb3eb on tablet 95b1058e6dd5473ca41d534137ebb3eb with peer 74506b6a243b450398cf31bdabf3eccf
I20240502 07:58:19.253806 21045 ts_tablet_manager.cc:902] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7: Initiating tablet copy from peer dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153)
I20240502 07:58:19.254235 21045 tablet_copy_client.cc:308] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:44153
I20240502 07:58:19.257218 19107 tablet_copy_service.cc:140] P dfc05317d0c04450af55ceabe36f6a99: Received BeginTabletCopySession request for tablet 12852f6b0d7242208c3f2beea408bc89 from peer 278943ee039c48988108fc425b366ff7 ({username='slave'} at 127.16.85.132:46339)
I20240502 07:58:19.257288 19107 tablet_copy_service.cc:161] P dfc05317d0c04450af55ceabe36f6a99: Beginning new tablet copy session on tablet 12852f6b0d7242208c3f2beea408bc89 from peer 278943ee039c48988108fc425b366ff7 at {username='slave'} at 127.16.85.132:46339: session id = 278943ee039c48988108fc425b366ff7-12852f6b0d7242208c3f2beea408bc89
I20240502 07:58:19.257894 19107 tablet_copy_source_session.cc:205] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:19.258579 21045 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 12852f6b0d7242208c3f2beea408bc89. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:19.260061 21045 tablet_copy_client.cc:783] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:19.260210 21045 tablet_copy_client.cc:647] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:19.261193 21045 tablet_copy_client.cc:523] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:19.261977 21045 tablet_bootstrap.cc:492] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7: Bootstrap starting.
I20240502 07:58:19.276456 21045 tablet_bootstrap.cc:492] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:19.276643 21045 tablet_bootstrap.cc:492] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7: Bootstrap complete.
I20240502 07:58:19.276782 21045 ts_tablet_manager.cc:1367] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7: Time spent bootstrapping tablet: real 0.015s user 0.011s sys 0.004s
I20240502 07:58:19.277165 21045 raft_consensus.cc:348] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: true } }
I20240502 07:58:19.277244 21045 raft_consensus.cc:729] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 278943ee039c48988108fc425b366ff7, State: Initialized, Role: LEARNER
I20240502 07:58:19.277335 21045 consensus_queue.cc:260] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: true } }
I20240502 07:58:19.277529 21045 ts_tablet_manager.cc:1398] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:19.278173 19107 tablet_copy_service.cc:342] P dfc05317d0c04450af55ceabe36f6a99: Request end of tablet copy session 278943ee039c48988108fc425b366ff7-12852f6b0d7242208c3f2beea408bc89 received from {username='slave'} at 127.16.85.132:46339
I20240502 07:58:19.278234 19107 tablet_copy_service.cc:433] P dfc05317d0c04450af55ceabe36f6a99: ending tablet copy session 278943ee039c48988108fc425b366ff7-12852f6b0d7242208c3f2beea408bc89 on tablet 12852f6b0d7242208c3f2beea408bc89 with peer 278943ee039c48988108fc425b366ff7
I20240502 07:58:19.432463 20611 raft_consensus.cc:1203] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:19.488807 20349 raft_consensus.cc:1203] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:19.504086 20218 raft_consensus.cc:1203] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:19.591903 20218 raft_consensus.cc:1203] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:19.612133 20480 raft_consensus.cc:1203] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:19.641005 20480 raft_consensus.cc:1203] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:19.751317 20349 raft_consensus.cc:1203] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:19.775270 20739 raft_consensus.cc:1203] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:19.830708 20611 raft_consensus.cc:1203] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:19.945780 21188 raft_consensus.cc:1050] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99: attempting to promote NON_VOTER cbc178a0a2f545658c5961645415eb25 to VOTER
I20240502 07:58:19.946187 21188 consensus_queue.cc:237] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } }
I20240502 07:58:19.947132 20218 raft_consensus.cc:1261] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [term 1 LEARNER]: Refusing update from remote peer dfc05317d0c04450af55ceabe36f6a99: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:19.947508 21187 consensus_queue.cc:1035] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:19.949606 21188 raft_consensus.cc:2932] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, cbc178a0a2f545658c5961645415eb25 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } } }
I20240502 07:58:19.950105 20218 raft_consensus.cc:2932] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, cbc178a0a2f545658c5961645415eb25 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } } }
I20240502 07:58:19.951689 18940 catalog_manager.cc:5484] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: config changed from index 4 to 5, cbc178a0a2f545658c5961645415eb25 (127.16.85.131) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:20.044726 21160 raft_consensus.cc:1050] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118: attempting to promote NON_VOTER 74506b6a243b450398cf31bdabf3eccf to VOTER
I20240502 07:58:20.045147 21160 consensus_queue.cc:237] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } }
I20240502 07:58:20.046110 20611 raft_consensus.cc:1261] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [term 1 LEARNER]: Refusing update from remote peer 3a68f86d2b9d44108fbb383fd0ec3118: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:20.046516 21160 consensus_queue.cc:1035] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:20.047363 21160 raft_consensus.cc:1050] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118: attempting to promote NON_VOTER 278943ee039c48988108fc425b366ff7 to VOTER
I20240502 07:58:20.047600 21160 consensus_queue.cc:237] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } }
I20240502 07:58:20.048594 20349 raft_consensus.cc:1261] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [term 1 LEARNER]: Refusing update from remote peer 3a68f86d2b9d44108fbb383fd0ec3118: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:20.048975 21161 consensus_queue.cc:1035] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Connected to new peer: Peer: permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:20.049124 21160 raft_consensus.cc:2932] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74506b6a243b450398cf31bdabf3eccf (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } } }
I20240502 07:58:20.049497 20611 raft_consensus.cc:2932] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74506b6a243b450398cf31bdabf3eccf (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } } }
I20240502 07:58:20.051609 18936 catalog_manager.cc:5484] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 reported cstate change: config changed from index 4 to 5, 74506b6a243b450398cf31bdabf3eccf (127.16.85.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:20.052270 21161 raft_consensus.cc:2932] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 278943ee039c48988108fc425b366ff7 (127.16.85.132) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } } }
I20240502 07:58:20.052716 20349 raft_consensus.cc:2932] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 278943ee039c48988108fc425b366ff7 (127.16.85.132) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } } }
I20240502 07:58:20.054832 18940 catalog_manager.cc:5484] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 reported cstate change: config changed from index 4 to 5, 278943ee039c48988108fc425b366ff7 (127.16.85.132) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } } }
I20240502 07:58:20.078612 21160 raft_consensus.cc:1050] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118: attempting to promote NON_VOTER 3a4ce39a207341a4883e10e934b839b4 to VOTER
I20240502 07:58:20.078869 21160 consensus_queue.cc:237] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } }
I20240502 07:58:20.079775 20480 raft_consensus.cc:1261] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [term 1 LEARNER]: Refusing update from remote peer 3a68f86d2b9d44108fbb383fd0ec3118: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:20.080108 21160 consensus_queue.cc:1035] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:20.082274 20930 raft_consensus.cc:2932] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3a4ce39a207341a4883e10e934b839b4 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } } }
I20240502 07:58:20.082710 20480 raft_consensus.cc:2932] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3a4ce39a207341a4883e10e934b839b4 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } } }
I20240502 07:58:20.084190 18936 catalog_manager.cc:5484] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 reported cstate change: config changed from index 4 to 5, 3a4ce39a207341a4883e10e934b839b4 (127.16.85.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:20.153275 19218 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "0cb1145d5f6343a1b205bb59221e95f4"
dest_uuid: "3a68f86d2b9d44108fbb383fd0ec3118"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:54870
I20240502 07:58:20.153427 19218 raft_consensus.cc:595] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:20.161029 20888 raft_consensus.cc:1050] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99: attempting to promote NON_VOTER 3a4ce39a207341a4883e10e934b839b4 to VOTER
I20240502 07:58:20.161350 20888 consensus_queue.cc:237] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } }
I20240502 07:58:20.162310 20480 raft_consensus.cc:1261] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [term 1 LEARNER]: Refusing update from remote peer dfc05317d0c04450af55ceabe36f6a99: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:20.162664 20888 consensus_queue.cc:1035] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:20.164276 21188 raft_consensus.cc:2932] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3a4ce39a207341a4883e10e934b839b4 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } } }
I20240502 07:58:20.164738 20480 raft_consensus.cc:2932] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3a4ce39a207341a4883e10e934b839b4 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } } }
I20240502 07:58:20.166330 18940 catalog_manager.cc:5484] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: config changed from index 4 to 5, 3a4ce39a207341a4883e10e934b839b4 (127.16.85.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:20.168830 19218 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "63baa0ba36f742ca91fddceb0e4b9617"
dest_uuid: "3a68f86d2b9d44108fbb383fd0ec3118"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:54880
I20240502 07:58:20.168926 19218 raft_consensus.cc:595] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:20.170122 20888 raft_consensus.cc:1050] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99: attempting to promote NON_VOTER cbc178a0a2f545658c5961645415eb25 to VOTER
I20240502 07:58:20.170361 20888 consensus_queue.cc:237] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } }
I20240502 07:58:20.171674 20218 raft_consensus.cc:1261] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [term 1 LEARNER]: Refusing update from remote peer dfc05317d0c04450af55ceabe36f6a99: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:20.172013 21187 consensus_queue.cc:1035] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:20.173944 20888 raft_consensus.cc:2932] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, cbc178a0a2f545658c5961645415eb25 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } } }
I20240502 07:58:20.174343 20218 raft_consensus.cc:2932] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, cbc178a0a2f545658c5961645415eb25 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } } }
I20240502 07:58:20.175837 18936 catalog_manager.cc:5484] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: config changed from index 4 to 5, cbc178a0a2f545658c5961645415eb25 (127.16.85.131) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:20.188910 19218 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "af6d015cb2154a29970c5a7a4116efe9"
dest_uuid: "3a68f86d2b9d44108fbb383fd0ec3118"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:54900
I20240502 07:58:20.189005 19218 raft_consensus.cc:595] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:20.196692 19087 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "f8550e00f0ba4b5ea52822f99add2cfe"
dest_uuid: "dfc05317d0c04450af55ceabe36f6a99"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40594
I20240502 07:58:20.196836 19087 raft_consensus.cc:595] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:20.211346 21188 raft_consensus.cc:1050] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99: attempting to promote NON_VOTER f81ce2156e854e0ab96f4b6fb2077714 to VOTER
I20240502 07:58:20.211606 21188 consensus_queue.cc:237] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: false } }
I20240502 07:58:20.212463 20739 raft_consensus.cc:1261] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [term 1 LEARNER]: Refusing update from remote peer dfc05317d0c04450af55ceabe36f6a99: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:20.212802 21188 consensus_queue.cc:1035] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:20.214344 20888 raft_consensus.cc:2932] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f81ce2156e854e0ab96f4b6fb2077714 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: false } } }
I20240502 07:58:20.214767 20739 raft_consensus.cc:2932] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, f81ce2156e854e0ab96f4b6fb2077714 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: false } } }
I20240502 07:58:20.216437 18940 catalog_manager.cc:5484] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: config changed from index 4 to 5, f81ce2156e854e0ab96f4b6fb2077714 (127.16.85.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:20.217813 21188 raft_consensus.cc:1050] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99: attempting to promote NON_VOTER 278943ee039c48988108fc425b366ff7 to VOTER
I20240502 07:58:20.218037 21188 consensus_queue.cc:237] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } }
I20240502 07:58:20.218883 20349 raft_consensus.cc:1261] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [term 1 LEARNER]: Refusing update from remote peer dfc05317d0c04450af55ceabe36f6a99: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:20.219161 20888 consensus_queue.cc:1035] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:20.220549 21188 raft_consensus.cc:2932] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 278943ee039c48988108fc425b366ff7 (127.16.85.132) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } } }
I20240502 07:58:20.221022 20349 raft_consensus.cc:2932] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 278943ee039c48988108fc425b366ff7 (127.16.85.132) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } } }
I20240502 07:58:20.222831 18940 catalog_manager.cc:5484] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: config changed from index 4 to 5, 278943ee039c48988108fc425b366ff7 (127.16.85.132) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:20.230131 21160 raft_consensus.cc:1050] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118: attempting to promote NON_VOTER 74506b6a243b450398cf31bdabf3eccf to VOTER
I20240502 07:58:20.230422 21160 consensus_queue.cc:237] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } }
I20240502 07:58:20.231299 20611 raft_consensus.cc:1261] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [term 1 LEARNER]: Refusing update from remote peer 3a68f86d2b9d44108fbb383fd0ec3118: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:20.231629 21160 consensus_queue.cc:1035] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:20.233235 21161 raft_consensus.cc:2932] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74506b6a243b450398cf31bdabf3eccf (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } } }
I20240502 07:58:20.233639 20611 raft_consensus.cc:2932] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74506b6a243b450398cf31bdabf3eccf (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } } }
I20240502 07:58:20.235380 18939 catalog_manager.cc:5484] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf reported cstate change: config changed from index 4 to 5, 74506b6a243b450398cf31bdabf3eccf (127.16.85.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } } }
I20240502 07:58:20.406247 19087 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "823e3d11a041417e90344de4350fee72"
dest_uuid: "dfc05317d0c04450af55ceabe36f6a99"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40610
I20240502 07:58:20.406361 19087 raft_consensus.cc:595] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:20.414680 19218 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "95b1058e6dd5473ca41d534137ebb3eb"
dest_uuid: "3a68f86d2b9d44108fbb383fd0ec3118"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:54914
I20240502 07:58:20.414781 19218 raft_consensus.cc:595] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:20.422724 19218 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "0cb1145d5f6343a1b205bb59221e95f4"
dest_uuid: "3a68f86d2b9d44108fbb383fd0ec3118"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:54930
I20240502 07:58:20.422827 19218 raft_consensus.cc:595] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:20.430999 19087 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "53f83cc7a58447158d0dae01fe31adba"
dest_uuid: "dfc05317d0c04450af55ceabe36f6a99"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40640
I20240502 07:58:20.431109 19087 raft_consensus.cc:595] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:20.438761 19218 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "63baa0ba36f742ca91fddceb0e4b9617"
dest_uuid: "3a68f86d2b9d44108fbb383fd0ec3118"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:54956
I20240502 07:58:20.438856 19218 raft_consensus.cc:595] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:20.440820 21160 raft_consensus.cc:979] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118: : Instructing follower 278943ee039c48988108fc425b366ff7 to start an election
I20240502 07:58:20.440920 21160 raft_consensus.cc:1067] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Signalling peer 278943ee039c48988108fc425b366ff7 to start an election
I20240502 07:58:20.441515 20349 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "af6d015cb2154a29970c5a7a4116efe9"
dest_uuid: "278943ee039c48988108fc425b366ff7"
from {username='slave'} at 127.16.85.130:54407
I20240502 07:58:20.441699 20349 raft_consensus.cc:482] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 07:58:20.441808 20349 raft_consensus.cc:3037] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.442555 20349 raft_consensus.cc:504] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } }
I20240502 07:58:20.443401 20349 leader_election.cc:288] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [CANDIDATE]: Term 2 election: Requested vote from peers 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130:33077)
I20240502 07:58:20.443773 19218 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af6d015cb2154a29970c5a7a4116efe9" candidate_uuid: "278943ee039c48988108fc425b366ff7" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "3a68f86d2b9d44108fbb383fd0ec3118"
I20240502 07:58:20.443882 19218 raft_consensus.cc:3032] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 07:58:20.443912 19218 raft_consensus.cc:729] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Running, Role: LEADER
I20240502 07:58:20.444020 19218 consensus_queue.cc:260] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } }
I20240502 07:58:20.444120 19218 raft_consensus.cc:3037] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.444782 19218 raft_consensus.cc:2454] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 278943ee039c48988108fc425b366ff7 in term 2.
I20240502 07:58:20.445101 20285 leader_election.cc:302] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 278943ee039c48988108fc425b366ff7, 3a68f86d2b9d44108fbb383fd0ec3118; no voters:
I20240502 07:58:20.445312 21062 raft_consensus.cc:2781] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:20.445466 21062 raft_consensus.cc:686] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [term 2 LEADER]: Becoming Leader. State: Replica: 278943ee039c48988108fc425b366ff7, State: Running, Role: LEADER
I20240502 07:58:20.445700 21062 consensus_queue.cc:237] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } }
I20240502 07:58:20.447266 18940 catalog_manager.cc:5484] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 reported cstate change: term changed from 1 to 2, leader changed from 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130) to 278943ee039c48988108fc425b366ff7 (127.16.85.132). New cstate: current_term: 2 leader_uuid: "278943ee039c48988108fc425b366ff7" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:20.449656 19087 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "708e377a224e40deab16180eb8f32c82"
dest_uuid: "dfc05317d0c04450af55ceabe36f6a99"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40670
I20240502 07:58:20.449774 19087 raft_consensus.cc:595] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Received request to transfer leadership
W20240502 07:58:20.454254 18930 catalog_manager.cc:5014] ChangeConfig:REMOVE_PEER RPC for tablet af6d015cb2154a29970c5a7a4116efe9 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER failed with leader 278943ee039c48988108fc425b366ff7 (127.16.85.132:44867) due to error UNKNOWN_ERROR; will retry: Illegal state: Leader has not yet committed an operation in its own term
I20240502 07:58:20.458220 19087 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "12852f6b0d7242208c3f2beea408bc89"
dest_uuid: "dfc05317d0c04450af55ceabe36f6a99"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40674
I20240502 07:58:20.458310 19087 raft_consensus.cc:595] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:20.470103 19087 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "f8550e00f0ba4b5ea52822f99add2cfe"
dest_uuid: "dfc05317d0c04450af55ceabe36f6a99"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40692
I20240502 07:58:20.470209 19087 raft_consensus.cc:595] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:20.505880 21160 raft_consensus.cc:979] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118: : Instructing follower 74506b6a243b450398cf31bdabf3eccf to start an election
I20240502 07:58:20.505992 21160 raft_consensus.cc:1067] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Signalling peer 74506b6a243b450398cf31bdabf3eccf to start an election
I20240502 07:58:20.506702 20611 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "0cb1145d5f6343a1b205bb59221e95f4"
dest_uuid: "74506b6a243b450398cf31bdabf3eccf"
from {username='slave'} at 127.16.85.130:59273
I20240502 07:58:20.506879 20611 raft_consensus.cc:482] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 07:58:20.506985 20611 raft_consensus.cc:3037] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.507789 20611 raft_consensus.cc:504] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } }
I20240502 07:58:20.508524 20611 leader_election.cc:288] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [CANDIDATE]: Term 2 election: Requested vote from peers 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130:33077)
I20240502 07:58:20.508862 19218 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0cb1145d5f6343a1b205bb59221e95f4" candidate_uuid: "74506b6a243b450398cf31bdabf3eccf" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "3a68f86d2b9d44108fbb383fd0ec3118"
I20240502 07:58:20.508960 19218 raft_consensus.cc:3032] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 07:58:20.508992 19218 raft_consensus.cc:729] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Running, Role: LEADER
I20240502 07:58:20.509109 19218 consensus_queue.cc:260] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } }
I20240502 07:58:20.509215 19218 raft_consensus.cc:3037] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.509827 19218 raft_consensus.cc:2454] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74506b6a243b450398cf31bdabf3eccf in term 2.
I20240502 07:58:20.510105 20547 leader_election.cc:302] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 3a68f86d2b9d44108fbb383fd0ec3118, 74506b6a243b450398cf31bdabf3eccf; no voters:
I20240502 07:58:20.510360 21040 raft_consensus.cc:2781] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:20.510550 21040 raft_consensus.cc:686] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [term 2 LEADER]: Becoming Leader. State: Replica: 74506b6a243b450398cf31bdabf3eccf, State: Running, Role: LEADER
I20240502 07:58:20.510823 21040 consensus_queue.cc:237] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } }
I20240502 07:58:20.511417 20888 raft_consensus.cc:979] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99: : Instructing follower cbc178a0a2f545658c5961645415eb25 to start an election
I20240502 07:58:20.511512 21187 raft_consensus.cc:1067] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Signalling peer cbc178a0a2f545658c5961645415eb25 to start an election
I20240502 07:58:20.512029 20218 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "f8550e00f0ba4b5ea52822f99add2cfe"
dest_uuid: "cbc178a0a2f545658c5961645415eb25"
from {username='slave'} at 127.16.85.129:54513
I20240502 07:58:20.512195 20218 raft_consensus.cc:482] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 07:58:20.512292 20218 raft_consensus.cc:3037] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.512678 18940 catalog_manager.cc:5484] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf reported cstate change: term changed from 1 to 2, leader changed from 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130) to 74506b6a243b450398cf31bdabf3eccf (127.16.85.134). New cstate: current_term: 2 leader_uuid: "74506b6a243b450398cf31bdabf3eccf" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:20.513227 20218 raft_consensus.cc:504] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } }
I20240502 07:58:20.513931 20218 leader_election.cc:288] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [CANDIDATE]: Term 2 election: Requested vote from peers dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153)
I20240502 07:58:20.514300 19087 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8550e00f0ba4b5ea52822f99add2cfe" candidate_uuid: "cbc178a0a2f545658c5961645415eb25" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "dfc05317d0c04450af55ceabe36f6a99"
I20240502 07:58:20.514447 19087 raft_consensus.cc:3032] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 07:58:20.514487 19087 raft_consensus.cc:729] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Running, Role: LEADER
I20240502 07:58:20.514612 19087 consensus_queue.cc:260] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } }
I20240502 07:58:20.514740 19087 raft_consensus.cc:3037] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.515657 19087 raft_consensus.cc:2454] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cbc178a0a2f545658c5961645415eb25 in term 2.
I20240502 07:58:20.515987 20153 leader_election.cc:302] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: cbc178a0a2f545658c5961645415eb25, dfc05317d0c04450af55ceabe36f6a99; no voters:
I20240502 07:58:20.516165 21061 raft_consensus.cc:2781] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:20.516311 21061 raft_consensus.cc:686] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [term 2 LEADER]: Becoming Leader. State: Replica: cbc178a0a2f545658c5961645415eb25, State: Running, Role: LEADER
I20240502 07:58:20.516522 21061 consensus_queue.cc:237] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } }
I20240502 07:58:20.517911 18940 catalog_manager.cc:5484] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 reported cstate change: term changed from 1 to 2, leader changed from dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) to cbc178a0a2f545658c5961645415eb25 (127.16.85.131). New cstate: current_term: 2 leader_uuid: "cbc178a0a2f545658c5961645415eb25" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:20.593827 20888 raft_consensus.cc:979] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99: : Instructing follower f81ce2156e854e0ab96f4b6fb2077714 to start an election
I20240502 07:58:20.593920 20888 raft_consensus.cc:1067] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Signalling peer f81ce2156e854e0ab96f4b6fb2077714 to start an election
I20240502 07:58:20.594381 20739 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "708e377a224e40deab16180eb8f32c82"
dest_uuid: "f81ce2156e854e0ab96f4b6fb2077714"
from {username='slave'} at 127.16.85.129:53299
I20240502 07:58:20.594547 20739 raft_consensus.cc:482] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 07:58:20.594592 20739 raft_consensus.cc:3037] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.595315 20739 raft_consensus.cc:504] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: false } }
I20240502 07:58:20.595968 20739 leader_election.cc:288] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [CANDIDATE]: Term 2 election: Requested vote from peers dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153)
I20240502 07:58:20.596323 19087 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "708e377a224e40deab16180eb8f32c82" candidate_uuid: "f81ce2156e854e0ab96f4b6fb2077714" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "dfc05317d0c04450af55ceabe36f6a99"
I20240502 07:58:20.596434 19087 raft_consensus.cc:3032] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 07:58:20.596480 19087 raft_consensus.cc:729] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Running, Role: LEADER
I20240502 07:58:20.596612 19087 consensus_queue.cc:260] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: false } }
I20240502 07:58:20.596763 19087 raft_consensus.cc:3037] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.597473 19087 raft_consensus.cc:2454] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f81ce2156e854e0ab96f4b6fb2077714 in term 2.
I20240502 07:58:20.597788 20677 leader_election.cc:302] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: dfc05317d0c04450af55ceabe36f6a99, f81ce2156e854e0ab96f4b6fb2077714; no voters:
I20240502 07:58:20.597963 21234 raft_consensus.cc:2781] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:20.598189 21234 raft_consensus.cc:686] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [term 2 LEADER]: Becoming Leader. State: Replica: f81ce2156e854e0ab96f4b6fb2077714, State: Running, Role: LEADER
I20240502 07:58:20.598327 21234 consensus_queue.cc:237] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: false } }
I20240502 07:58:20.599818 18940 catalog_manager.cc:5484] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 reported cstate change: term changed from 1 to 2, leader changed from dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) to f81ce2156e854e0ab96f4b6fb2077714 (127.16.85.135). New cstate: current_term: 2 leader_uuid: "f81ce2156e854e0ab96f4b6fb2077714" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:20.625779 21161 raft_consensus.cc:979] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118: : Instructing follower 74506b6a243b450398cf31bdabf3eccf to start an election
I20240502 07:58:20.625876 21161 raft_consensus.cc:1067] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Signalling peer 74506b6a243b450398cf31bdabf3eccf to start an election
I20240502 07:58:20.626281 20611 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "95b1058e6dd5473ca41d534137ebb3eb"
dest_uuid: "74506b6a243b450398cf31bdabf3eccf"
from {username='slave'} at 127.16.85.130:59273
I20240502 07:58:20.626395 20611 raft_consensus.cc:482] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 07:58:20.626451 20611 raft_consensus.cc:3037] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.627161 20611 raft_consensus.cc:504] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } }
I20240502 07:58:20.627432 20611 leader_election.cc:288] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [CANDIDATE]: Term 2 election: Requested vote from peers 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130:33077)
I20240502 07:58:20.627781 19218 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "95b1058e6dd5473ca41d534137ebb3eb" candidate_uuid: "74506b6a243b450398cf31bdabf3eccf" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "3a68f86d2b9d44108fbb383fd0ec3118"
I20240502 07:58:20.627887 19218 raft_consensus.cc:3032] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 07:58:20.627941 19218 raft_consensus.cc:729] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Running, Role: LEADER
I20240502 07:58:20.628085 19218 consensus_queue.cc:260] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } }
I20240502 07:58:20.628217 19218 raft_consensus.cc:3037] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.628907 19218 raft_consensus.cc:2454] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74506b6a243b450398cf31bdabf3eccf in term 2.
I20240502 07:58:20.629156 20547 leader_election.cc:302] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 3a68f86d2b9d44108fbb383fd0ec3118, 74506b6a243b450398cf31bdabf3eccf; no voters:
I20240502 07:58:20.629318 21040 raft_consensus.cc:2781] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:20.629428 21040 raft_consensus.cc:686] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [term 2 LEADER]: Becoming Leader. State: Replica: 74506b6a243b450398cf31bdabf3eccf, State: Running, Role: LEADER
I20240502 07:58:20.629547 21040 consensus_queue.cc:237] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } }
I20240502 07:58:20.630772 18940 catalog_manager.cc:5484] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf reported cstate change: term changed from 1 to 2, leader changed from 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130) to 74506b6a243b450398cf31bdabf3eccf (127.16.85.134). New cstate: current_term: 2 leader_uuid: "74506b6a243b450398cf31bdabf3eccf" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:20.656047 21160 raft_consensus.cc:979] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118: : Instructing follower 3a4ce39a207341a4883e10e934b839b4 to start an election
I20240502 07:58:20.656143 21160 raft_consensus.cc:1067] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Signalling peer 3a4ce39a207341a4883e10e934b839b4 to start an election
I20240502 07:58:20.656512 20480 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "63baa0ba36f742ca91fddceb0e4b9617"
dest_uuid: "3a4ce39a207341a4883e10e934b839b4"
from {username='slave'} at 127.16.85.130:40709
I20240502 07:58:20.656708 20480 raft_consensus.cc:482] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 07:58:20.656806 20480 raft_consensus.cc:3037] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.657653 20480 raft_consensus.cc:504] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } }
I20240502 07:58:20.658516 20480 leader_election.cc:288] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [CANDIDATE]: Term 2 election: Requested vote from peers 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130:33077)
I20240502 07:58:20.658844 19218 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "63baa0ba36f742ca91fddceb0e4b9617" candidate_uuid: "3a4ce39a207341a4883e10e934b839b4" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "3a68f86d2b9d44108fbb383fd0ec3118"
I20240502 07:58:20.658967 19218 raft_consensus.cc:3032] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 07:58:20.659011 19218 raft_consensus.cc:729] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Running, Role: LEADER
I20240502 07:58:20.659107 19218 consensus_queue.cc:260] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } }
I20240502 07:58:20.659204 19218 raft_consensus.cc:3037] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.659840 19218 raft_consensus.cc:2454] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3a4ce39a207341a4883e10e934b839b4 in term 2.
I20240502 07:58:20.660151 20416 leader_election.cc:302] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 3a4ce39a207341a4883e10e934b839b4, 3a68f86d2b9d44108fbb383fd0ec3118; no voters:
I20240502 07:58:20.660485 21092 raft_consensus.cc:2781] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:20.660697 21092 raft_consensus.cc:686] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [term 2 LEADER]: Becoming Leader. State: Replica: 3a4ce39a207341a4883e10e934b839b4, State: Running, Role: LEADER
I20240502 07:58:20.660980 21092 consensus_queue.cc:237] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } }
I20240502 07:58:20.661234 20888 raft_consensus.cc:979] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99: : Instructing follower 278943ee039c48988108fc425b366ff7 to start an election
I20240502 07:58:20.661321 20888 raft_consensus.cc:1067] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Signalling peer 278943ee039c48988108fc425b366ff7 to start an election
I20240502 07:58:20.661733 20349 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "12852f6b0d7242208c3f2beea408bc89"
dest_uuid: "278943ee039c48988108fc425b366ff7"
from {username='slave'} at 127.16.85.129:54765
I20240502 07:58:20.661849 20349 raft_consensus.cc:482] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 07:58:20.661906 20349 raft_consensus.cc:3037] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.662684 20349 raft_consensus.cc:504] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } }
I20240502 07:58:20.662736 18940 catalog_manager.cc:5484] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 reported cstate change: term changed from 1 to 2, leader changed from 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130) to 3a4ce39a207341a4883e10e934b839b4 (127.16.85.133). New cstate: current_term: 2 leader_uuid: "3a4ce39a207341a4883e10e934b839b4" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:20.662993 20349 leader_election.cc:288] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [CANDIDATE]: Term 2 election: Requested vote from peers dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153)
I20240502 07:58:20.663362 19087 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "12852f6b0d7242208c3f2beea408bc89" candidate_uuid: "278943ee039c48988108fc425b366ff7" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "dfc05317d0c04450af55ceabe36f6a99"
I20240502 07:58:20.663457 19087 raft_consensus.cc:3032] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 07:58:20.663499 19087 raft_consensus.cc:729] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Running, Role: LEADER
I20240502 07:58:20.663632 19087 consensus_queue.cc:260] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } }
I20240502 07:58:20.663740 19087 raft_consensus.cc:3037] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.664592 19087 raft_consensus.cc:2454] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 278943ee039c48988108fc425b366ff7 in term 2.
I20240502 07:58:20.664865 20284 leader_election.cc:302] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 278943ee039c48988108fc425b366ff7, dfc05317d0c04450af55ceabe36f6a99; no voters:
I20240502 07:58:20.665086 21062 raft_consensus.cc:2781] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:20.665169 21062 raft_consensus.cc:686] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [term 2 LEADER]: Becoming Leader. State: Replica: 278943ee039c48988108fc425b366ff7, State: Running, Role: LEADER
I20240502 07:58:20.665293 21062 consensus_queue.cc:237] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } }
I20240502 07:58:20.666349 18940 catalog_manager.cc:5484] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 reported cstate change: term changed from 1 to 2, leader changed from dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) to 278943ee039c48988108fc425b366ff7 (127.16.85.132). New cstate: current_term: 2 leader_uuid: "278943ee039c48988108fc425b366ff7" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:20.679351 19087 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "823e3d11a041417e90344de4350fee72"
dest_uuid: "dfc05317d0c04450af55ceabe36f6a99"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40710
I20240502 07:58:20.679445 19087 raft_consensus.cc:595] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:20.680188 21187 raft_consensus.cc:979] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99: : Instructing follower cbc178a0a2f545658c5961645415eb25 to start an election
I20240502 07:58:20.680269 21187 raft_consensus.cc:1067] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Signalling peer cbc178a0a2f545658c5961645415eb25 to start an election
I20240502 07:58:20.680739 20218 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "53f83cc7a58447158d0dae01fe31adba"
dest_uuid: "cbc178a0a2f545658c5961645415eb25"
from {username='slave'} at 127.16.85.129:54513
I20240502 07:58:20.680863 20218 raft_consensus.cc:482] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 07:58:20.680925 20218 raft_consensus.cc:3037] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.681902 20218 raft_consensus.cc:504] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } }
I20240502 07:58:20.682147 20218 leader_election.cc:288] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [CANDIDATE]: Term 2 election: Requested vote from peers dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153)
I20240502 07:58:20.682484 19087 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "53f83cc7a58447158d0dae01fe31adba" candidate_uuid: "cbc178a0a2f545658c5961645415eb25" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "dfc05317d0c04450af55ceabe36f6a99"
I20240502 07:58:20.682580 19087 raft_consensus.cc:3032] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 07:58:20.682623 19087 raft_consensus.cc:729] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Running, Role: LEADER
I20240502 07:58:20.682745 19087 consensus_queue.cc:260] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } }
I20240502 07:58:20.682866 19087 raft_consensus.cc:3037] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.683526 19087 raft_consensus.cc:2454] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cbc178a0a2f545658c5961645415eb25 in term 2.
I20240502 07:58:20.683766 20153 leader_election.cc:302] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: cbc178a0a2f545658c5961645415eb25, dfc05317d0c04450af55ceabe36f6a99; no voters:
I20240502 07:58:20.683913 21061 raft_consensus.cc:2781] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:20.684048 21061 raft_consensus.cc:686] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [term 2 LEADER]: Becoming Leader. State: Replica: cbc178a0a2f545658c5961645415eb25, State: Running, Role: LEADER
I20240502 07:58:20.684173 21061 consensus_queue.cc:237] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } }
I20240502 07:58:20.685592 18940 catalog_manager.cc:5484] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 reported cstate change: term changed from 1 to 2, leader changed from dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) to cbc178a0a2f545658c5961645415eb25 (127.16.85.131). New cstate: current_term: 2 leader_uuid: "cbc178a0a2f545658c5961645415eb25" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:20.768293 21187 raft_consensus.cc:979] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99: : Instructing follower 3a4ce39a207341a4883e10e934b839b4 to start an election
I20240502 07:58:20.768399 21187 raft_consensus.cc:1067] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Signalling peer 3a4ce39a207341a4883e10e934b839b4 to start an election
I20240502 07:58:20.768782 20480 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "823e3d11a041417e90344de4350fee72"
dest_uuid: "3a4ce39a207341a4883e10e934b839b4"
from {username='slave'} at 127.16.85.129:48497
I20240502 07:58:20.768894 20480 raft_consensus.cc:482] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 07:58:20.768967 20480 raft_consensus.cc:3037] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.769671 20480 raft_consensus.cc:504] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } }
I20240502 07:58:20.769994 20480 leader_election.cc:288] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [CANDIDATE]: Term 2 election: Requested vote from peers dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153)
I20240502 07:58:20.770401 19087 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "823e3d11a041417e90344de4350fee72" candidate_uuid: "3a4ce39a207341a4883e10e934b839b4" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "dfc05317d0c04450af55ceabe36f6a99"
I20240502 07:58:20.770505 19087 raft_consensus.cc:3032] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 07:58:20.770540 19087 raft_consensus.cc:729] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Running, Role: LEADER
I20240502 07:58:20.770660 19087 consensus_queue.cc:260] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } }
I20240502 07:58:20.770771 19087 raft_consensus.cc:3037] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:20.771440 19087 raft_consensus.cc:2454] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3a4ce39a207341a4883e10e934b839b4 in term 2.
I20240502 07:58:20.771705 20415 leader_election.cc:302] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 3a4ce39a207341a4883e10e934b839b4, dfc05317d0c04450af55ceabe36f6a99; no voters:
I20240502 07:58:20.771842 21092 raft_consensus.cc:2781] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:20.771914 21092 raft_consensus.cc:686] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [term 2 LEADER]: Becoming Leader. State: Replica: 3a4ce39a207341a4883e10e934b839b4, State: Running, Role: LEADER
I20240502 07:58:20.772032 21092 consensus_queue.cc:237] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } }
I20240502 07:58:20.773172 18940 catalog_manager.cc:5484] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 reported cstate change: term changed from 1 to 2, leader changed from dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) to 3a4ce39a207341a4883e10e934b839b4 (127.16.85.133). New cstate: current_term: 2 leader_uuid: "3a4ce39a207341a4883e10e934b839b4" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:20.878700 19218 raft_consensus.cc:1261] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Refusing update from remote peer 278943ee039c48988108fc425b366ff7: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 07:58:20.879235 21062 consensus_queue.cc:1035] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:20.991889 19087 raft_consensus.cc:1261] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Refusing update from remote peer cbc178a0a2f545658c5961645415eb25: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 07:58:20.992419 21061 consensus_queue.cc:1035] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:21.034828 19087 raft_consensus.cc:1261] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Refusing update from remote peer f81ce2156e854e0ab96f4b6fb2077714: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 07:58:21.035270 21234 consensus_queue.cc:1035] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:21.099851 20348 consensus_queue.cc:237] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } }
I20240502 07:58:21.100505 21422 raft_consensus.cc:2932] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130) evicted. New config: { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } } }
I20240502 07:58:21.101836 18930 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet af6d015cb2154a29970c5a7a4116efe9 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 07:58:21.102649 18940 catalog_manager.cc:5484] T af6d015cb2154a29970c5a7a4116efe9 P 278943ee039c48988108fc425b366ff7 reported cstate change: config changed from index 5 to 7, VOTER 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130) evicted. New cstate: current_term: 2 leader_uuid: "278943ee039c48988108fc425b366ff7" committed_config { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:21.104854 19197 tablet_service.cc:1514] Processing DeleteTablet for tablet af6d015cb2154a29970c5a7a4116efe9 with delete_type TABLET_DATA_TOMBSTONED (TS 3a68f86d2b9d44108fbb383fd0ec3118 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:59140
I20240502 07:58:21.105264 21564 tablet_replica.cc:330] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118: stopping tablet replica
I20240502 07:58:21.105499 21564 raft_consensus.cc:2229] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:21.105612 21564 raft_consensus.cc:2258] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:21.106449 21564 ts_tablet_manager.cc:1875] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:21.108186 21564 ts_tablet_manager.cc:1888] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 07:58:21.108256 21564 log.cc:1196] T af6d015cb2154a29970c5a7a4116efe9 P 3a68f86d2b9d44108fbb383fd0ec3118: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/af6d015cb2154a29970c5a7a4116efe9
I20240502 07:58:21.108718 18932 catalog_manager.cc:4830] TS 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130:33077): tablet af6d015cb2154a29970c5a7a4116efe9 (table rebalance_test_table_1 [id=cf7c2ac2aad3476c911268e8abb1e143]) successfully deleted
I20240502 07:58:21.114830 19218 raft_consensus.cc:1261] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Refusing update from remote peer 74506b6a243b450398cf31bdabf3eccf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 07:58:21.115237 21040 consensus_queue.cc:1035] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:21.125907 19218 raft_consensus.cc:1261] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Refusing update from remote peer 74506b6a243b450398cf31bdabf3eccf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 07:58:21.126240 21040 consensus_queue.cc:1035] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:21.131654 19087 raft_consensus.cc:1261] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Refusing update from remote peer 278943ee039c48988108fc425b366ff7: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 07:58:21.132025 21062 consensus_queue.cc:1035] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:21.171339 20611 consensus_queue.cc:237] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } }
I20240502 07:58:21.171970 21040 raft_consensus.cc:2932] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130) evicted. New config: { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } } }
I20240502 07:58:21.173215 18932 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0cb1145d5f6343a1b205bb59221e95f4 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 07:58:21.174410 18939 catalog_manager.cc:5484] T 0cb1145d5f6343a1b205bb59221e95f4 P 74506b6a243b450398cf31bdabf3eccf reported cstate change: config changed from index 5 to 7, VOTER 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130) evicted. New cstate: current_term: 2 leader_uuid: "74506b6a243b450398cf31bdabf3eccf" committed_config { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:21.176746 19197 tablet_service.cc:1514] Processing DeleteTablet for tablet 0cb1145d5f6343a1b205bb59221e95f4 with delete_type TABLET_DATA_TOMBSTONED (TS 3a68f86d2b9d44108fbb383fd0ec3118 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:59140
I20240502 07:58:21.176949 21564 tablet_replica.cc:330] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118: stopping tablet replica
I20240502 07:58:21.177070 21564 raft_consensus.cc:2229] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:21.177167 21564 raft_consensus.cc:2258] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:21.177783 21564 ts_tablet_manager.cc:1875] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:21.179628 20218 consensus_queue.cc:237] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } }
I20240502 07:58:21.180230 21061 raft_consensus.cc:2932] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) evicted. New config: { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } } }
I20240502 07:58:21.207262 19087 raft_consensus.cc:1261] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Refusing update from remote peer 3a4ce39a207341a4883e10e934b839b4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 07:58:21.207695 21092 consensus_queue.cc:1035] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:21.219866 18931 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f8550e00f0ba4b5ea52822f99add2cfe with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 07:58:21.220525 21564 ts_tablet_manager.cc:1888] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 07:58:21.220609 21564 log.cc:1196] T 0cb1145d5f6343a1b205bb59221e95f4 P 3a68f86d2b9d44108fbb383fd0ec3118: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/0cb1145d5f6343a1b205bb59221e95f4
I20240502 07:58:21.220894 18940 catalog_manager.cc:5484] T f8550e00f0ba4b5ea52822f99add2cfe P cbc178a0a2f545658c5961645415eb25 reported cstate change: config changed from index 5 to 7, VOTER dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) evicted. New cstate: current_term: 2 leader_uuid: "cbc178a0a2f545658c5961645415eb25" committed_config { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:21.221203 18932 catalog_manager.cc:4830] TS 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130:33077): tablet 0cb1145d5f6343a1b205bb59221e95f4 (table rebalance_test_table_3 [id=44e95bee069c42d2b54ac8810ce8424f]) successfully deleted
I20240502 07:58:21.221729 19218 raft_consensus.cc:1261] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Refusing update from remote peer 3a4ce39a207341a4883e10e934b839b4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 07:58:21.222110 21092 consensus_queue.cc:1035] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:21.223721 19066 tablet_service.cc:1514] Processing DeleteTablet for tablet f8550e00f0ba4b5ea52822f99add2cfe with delete_type TABLET_DATA_TOMBSTONED (TS dfc05317d0c04450af55ceabe36f6a99 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:36952
I20240502 07:58:21.224251 21620 tablet_replica.cc:330] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99: stopping tablet replica
I20240502 07:58:21.224622 21620 raft_consensus.cc:2229] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:21.225008 21620 raft_consensus.cc:2258] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:21.226233 21620 ts_tablet_manager.cc:1875] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:21.235085 20611 consensus_queue.cc:237] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } }
I20240502 07:58:21.235715 21455 raft_consensus.cc:2932] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130) evicted. New config: { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } } }
I20240502 07:58:21.256556 20739 consensus_queue.cc:237] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: false } }
I20240502 07:58:21.257151 21459 raft_consensus.cc:2932] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) evicted. New config: { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: false } } }
I20240502 07:58:21.263643 20480 consensus_queue.cc:237] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } }
I20240502 07:58:21.264163 21092 raft_consensus.cc:2932] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130) evicted. New config: { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } } }
I20240502 07:58:21.272747 20348 consensus_queue.cc:237] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } }
I20240502 07:58:21.273262 21062 raft_consensus.cc:2932] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) evicted. New config: { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } } }
I20240502 07:58:21.289803 19087 raft_consensus.cc:1261] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Refusing update from remote peer cbc178a0a2f545658c5961645415eb25: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 07:58:21.290205 21061 consensus_queue.cc:1035] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:21.293073 18931 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 708e377a224e40deab16180eb8f32c82 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 07:58:21.293654 18930 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 12852f6b0d7242208c3f2beea408bc89 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 07:58:21.293656 18940 catalog_manager.cc:5484] T 708e377a224e40deab16180eb8f32c82 P f81ce2156e854e0ab96f4b6fb2077714 reported cstate change: config changed from index 5 to 7, VOTER dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) evicted. New cstate: current_term: 2 leader_uuid: "f81ce2156e854e0ab96f4b6fb2077714" committed_config { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "f81ce2156e854e0ab96f4b6fb2077714" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 45589 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:21.293900 18932 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 63baa0ba36f742ca91fddceb0e4b9617 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 07:58:21.294057 18936 catalog_manager.cc:5484] T 12852f6b0d7242208c3f2beea408bc89 P 278943ee039c48988108fc425b366ff7 reported cstate change: config changed from index 5 to 7, VOTER dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) evicted. New cstate: current_term: 2 leader_uuid: "278943ee039c48988108fc425b366ff7" committed_config { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "278943ee039c48988108fc425b366ff7" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 44867 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:21.294612 18943 catalog_manager.cc:5484] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a4ce39a207341a4883e10e934b839b4 reported cstate change: config changed from index 5 to 7, VOTER 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130) evicted. New cstate: current_term: 2 leader_uuid: "3a4ce39a207341a4883e10e934b839b4" committed_config { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:21.294749 18941 catalog_manager.cc:5484] T 95b1058e6dd5473ca41d534137ebb3eb P 74506b6a243b450398cf31bdabf3eccf reported cstate change: config changed from index 5 to 7, VOTER 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130) evicted. New cstate: current_term: 2 leader_uuid: "74506b6a243b450398cf31bdabf3eccf" committed_config { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "74506b6a243b450398cf31bdabf3eccf" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 36697 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:21.295434 18932 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 95b1058e6dd5473ca41d534137ebb3eb with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 07:58:21.295722 21620 ts_tablet_manager.cc:1888] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 07:58:21.295814 21620 log.cc:1196] T f8550e00f0ba4b5ea52822f99add2cfe P dfc05317d0c04450af55ceabe36f6a99: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/f8550e00f0ba4b5ea52822f99add2cfe
I20240502 07:58:21.296509 18931 catalog_manager.cc:4830] TS dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153): tablet f8550e00f0ba4b5ea52822f99add2cfe (table rebalance_test_table_0 [id=9a3705a54c6e44889df8c4a5749fffe2]) successfully deleted
I20240502 07:58:21.297842 19066 tablet_service.cc:1514] Processing DeleteTablet for tablet 708e377a224e40deab16180eb8f32c82 with delete_type TABLET_DATA_TOMBSTONED (TS dfc05317d0c04450af55ceabe36f6a99 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:36952
I20240502 07:58:21.298527 19066 tablet_service.cc:1514] Processing DeleteTablet for tablet 12852f6b0d7242208c3f2beea408bc89 with delete_type TABLET_DATA_TOMBSTONED (TS dfc05317d0c04450af55ceabe36f6a99 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:36952
I20240502 07:58:21.298555 21620 tablet_replica.cc:330] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99: stopping tablet replica
I20240502 07:58:21.298719 21620 raft_consensus.cc:2229] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:21.298816 21620 raft_consensus.cc:2258] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:21.298924 19197 tablet_service.cc:1514] Processing DeleteTablet for tablet 63baa0ba36f742ca91fddceb0e4b9617 with delete_type TABLET_DATA_TOMBSTONED (TS 3a68f86d2b9d44108fbb383fd0ec3118 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:59140
I20240502 07:58:21.299114 19198 tablet_service.cc:1514] Processing DeleteTablet for tablet 95b1058e6dd5473ca41d534137ebb3eb with delete_type TABLET_DATA_TOMBSTONED (TS 3a68f86d2b9d44108fbb383fd0ec3118 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:59140
I20240502 07:58:21.299257 21564 tablet_replica.cc:330] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118: stopping tablet replica
I20240502 07:58:21.299391 21564 raft_consensus.cc:2229] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:21.299495 21564 raft_consensus.cc:2258] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:21.299497 21620 ts_tablet_manager.cc:1875] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:21.300016 21564 ts_tablet_manager.cc:1875] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:21.302131 20218 consensus_queue.cc:237] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } }
I20240502 07:58:21.302610 21457 raft_consensus.cc:2932] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) evicted. New config: { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } } }
I20240502 07:58:21.370687 21564 ts_tablet_manager.cc:1888] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 07:58:21.370791 21564 log.cc:1196] T 63baa0ba36f742ca91fddceb0e4b9617 P 3a68f86d2b9d44108fbb383fd0ec3118: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/63baa0ba36f742ca91fddceb0e4b9617
I20240502 07:58:21.370975 21620 ts_tablet_manager.cc:1888] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 07:58:21.371069 21620 log.cc:1196] T 708e377a224e40deab16180eb8f32c82 P dfc05317d0c04450af55ceabe36f6a99: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/708e377a224e40deab16180eb8f32c82
I20240502 07:58:21.371250 21564 tablet_replica.cc:330] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118: stopping tablet replica
I20240502 07:58:21.371387 21564 raft_consensus.cc:2229] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:21.371492 21564 raft_consensus.cc:2258] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:21.371377 18941 catalog_manager.cc:5484] T 53f83cc7a58447158d0dae01fe31adba P cbc178a0a2f545658c5961645415eb25 reported cstate change: config changed from index 5 to 7, VOTER dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) evicted. New cstate: current_term: 2 leader_uuid: "cbc178a0a2f545658c5961645415eb25" committed_config { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "cbc178a0a2f545658c5961645415eb25" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 42907 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:21.371667 18931 catalog_manager.cc:4830] TS dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153): tablet 708e377a224e40deab16180eb8f32c82 (table rebalance_test_table_3 [id=44e95bee069c42d2b54ac8810ce8424f]) successfully deleted
I20240502 07:58:21.371753 21620 tablet_replica.cc:330] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99: stopping tablet replica
I20240502 07:58:21.371804 18931 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 53f83cc7a58447158d0dae01fe31adba with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 07:58:21.371897 21620 raft_consensus.cc:2229] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:21.372007 21620 raft_consensus.cc:2258] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:21.372273 18932 catalog_manager.cc:4830] TS 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130:33077): tablet 63baa0ba36f742ca91fddceb0e4b9617 (table rebalance_test_table_4 [id=db78e3e5980d48a3886a4b5403848cb3]) successfully deleted
I20240502 07:58:21.372839 21564 ts_tablet_manager.cc:1875] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:21.373214 21620 ts_tablet_manager.cc:1875] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:21.374300 19066 tablet_service.cc:1514] Processing DeleteTablet for tablet 53f83cc7a58447158d0dae01fe31adba with delete_type TABLET_DATA_TOMBSTONED (TS dfc05317d0c04450af55ceabe36f6a99 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:36952
I20240502 07:58:21.414510 20480 consensus_queue.cc:237] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } }
I20240502 07:58:21.415117 21461 raft_consensus.cc:2932] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) evicted. New config: { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } } }
I20240502 07:58:21.467499 18932 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 823e3d11a041417e90344de4350fee72 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 07:58:21.468317 18941 catalog_manager.cc:5484] T 823e3d11a041417e90344de4350fee72 P 3a4ce39a207341a4883e10e934b839b4 reported cstate change: config changed from index 5 to 7, VOTER dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) evicted. New cstate: current_term: 2 leader_uuid: "3a4ce39a207341a4883e10e934b839b4" committed_config { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "3a4ce39a207341a4883e10e934b839b4" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 36251 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:21.470602 21620 ts_tablet_manager.cc:1888] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 07:58:21.470682 21620 log.cc:1196] T 12852f6b0d7242208c3f2beea408bc89 P dfc05317d0c04450af55ceabe36f6a99: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/12852f6b0d7242208c3f2beea408bc89
I20240502 07:58:21.470328 21564 ts_tablet_manager.cc:1888] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 07:58:21.470835 21564 log.cc:1196] T 95b1058e6dd5473ca41d534137ebb3eb P 3a68f86d2b9d44108fbb383fd0ec3118: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/95b1058e6dd5473ca41d534137ebb3eb
I20240502 07:58:21.471132 21620 tablet_replica.cc:330] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99: stopping tablet replica
I20240502 07:58:21.471251 21620 raft_consensus.cc:2229] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:21.471343 21620 raft_consensus.cc:2258] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:21.471315 19066 tablet_service.cc:1514] Processing DeleteTablet for tablet 823e3d11a041417e90344de4350fee72 with delete_type TABLET_DATA_TOMBSTONED (TS dfc05317d0c04450af55ceabe36f6a99 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:36952
I20240502 07:58:21.471874 18932 catalog_manager.cc:4830] TS 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130:33077): tablet 95b1058e6dd5473ca41d534137ebb3eb (table rebalance_test_table_2 [id=19022d6115e942f19e6e4f2c19bc5100]) successfully deleted
I20240502 07:58:21.471940 18931 catalog_manager.cc:4830] TS dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153): tablet 12852f6b0d7242208c3f2beea408bc89 (table rebalance_test_table_2 [id=19022d6115e942f19e6e4f2c19bc5100]) successfully deleted
I20240502 07:58:21.472256 21620 ts_tablet_manager.cc:1875] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:21.523135 21620 ts_tablet_manager.cc:1888] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 07:58:21.523255 21620 log.cc:1196] T 53f83cc7a58447158d0dae01fe31adba P dfc05317d0c04450af55ceabe36f6a99: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/53f83cc7a58447158d0dae01fe31adba
I20240502 07:58:21.523697 21620 tablet_replica.cc:330] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99: stopping tablet replica
I20240502 07:58:21.523726 18931 catalog_manager.cc:4830] TS dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153): tablet 53f83cc7a58447158d0dae01fe31adba (table rebalance_test_table_4 [id=db78e3e5980d48a3886a4b5403848cb3]) successfully deleted
I20240502 07:58:21.523839 21620 raft_consensus.cc:2229] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:21.523962 21620 raft_consensus.cc:2258] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:21.524637 21620 ts_tablet_manager.cc:1875] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:21.526629 21620 ts_tablet_manager.cc:1888] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 07:58:21.526696 21620 log.cc:1196] T 823e3d11a041417e90344de4350fee72 P dfc05317d0c04450af55ceabe36f6a99: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/823e3d11a041417e90344de4350fee72
I20240502 07:58:21.527159 18931 catalog_manager.cc:4830] TS dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153): tablet 823e3d11a041417e90344de4350fee72 (table rebalance_test_table_0 [id=9a3705a54c6e44889df8c4a5749fffe2]) successfully deleted
I20240502 07:58:21.545444 20460 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.548074 19198 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 07:58:21.548772 20591 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.549065 20329 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.550844 20198 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.568393 20713 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.582424 19066 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 07:58:21.652479 20329 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.652750 20460 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.653149 19198 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 07:58:21.657300 20713 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.661710 20198 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.662315 19066 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 07:58:21.681452 20591 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.865343 20460 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.865903 20198 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.868257 20329 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.873440 19198 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 07:58:21.874871 20713 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.889364 20591 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.894965 19066 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 07:58:21.977726 20198 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.977830 20329 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.979032 19066 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 07:58:21.982829 20460 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.988687 20591 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:21.989027 19198 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 07:58:22.005127 20713 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:22.084533 20460 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:22.084553 20329 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:22.087507 20198 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:22.089316 20591 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:22.094185 19198 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 07:58:22.099131 20713 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:22.113868 19066 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
2229511e8632482d8a20ec64c9c6ea56 | 127.16.85.190:41469 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
allow_unsafe_replication_factor | true | runtime,unsafe | all 1 server(s) checked
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.16.85.148:33323 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+---------+----------+----------------+-----------------
278943ee039c48988108fc425b366ff7 | 127.16.85.132:44867 | HEALTHY | <none> | 2 | 0
3a4ce39a207341a4883e10e934b839b4 | 127.16.85.133:36251 | HEALTHY | <none> | 2 | 0
3a68f86d2b9d44108fbb383fd0ec3118 | 127.16.85.130:33077 | HEALTHY | <none> | 3 | 0
74506b6a243b450398cf31bdabf3eccf | 127.16.85.134:36697 | HEALTHY | <none> | 2 | 0
cbc178a0a2f545658c5961645415eb25 | 127.16.85.131:42907 | HEALTHY | <none> | 2 | 0
dfc05317d0c04450af55ceabe36f6a99 | 127.16.85.129:44153 | HEALTHY | <none> | 3 | 0
f81ce2156e854e0ab96f4b6fb2077714 | 127.16.85.135:45589 | HEALTHY | <none> | 2 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 7
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 7 server(s) checked
local_ip_for_outbound_sockets | 127.16.85.129 | experimental | 127.16.85.129:44153
local_ip_for_outbound_sockets | 127.16.85.130 | experimental | 127.16.85.130:33077
local_ip_for_outbound_sockets | 127.16.85.131 | experimental | 127.16.85.131:42907
local_ip_for_outbound_sockets | 127.16.85.132 | experimental | 127.16.85.132:44867
local_ip_for_outbound_sockets | 127.16.85.133 | experimental | 127.16.85.133:36251
local_ip_for_outbound_sockets | 127.16.85.134 | experimental | 127.16.85.134:36697
local_ip_for_outbound_sockets | 127.16.85.135 | experimental | 127.16.85.135:45589
never_fsync | true | unsafe,advanced | all 7 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 7 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 7 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 7 server(s) checked
server_dump_info_format | pb | hidden | all 7 server(s) checked
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.16.85.129:44153
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.16.85.130:33077
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.16.85.131:42907
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.16.85.132:44867
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.16.85.133:36251
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.16.85.134:36697
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/info.pb | hidden | 127.16.85.135:45589
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.16.85.148:33323 | all 7 server(s) checked
time_source | builtin | all 7 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 8 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
TestTable | 1 | HEALTHY | 1 | 1 | 0 | 0 | 0
rebalance_test_table_0 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_4 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 2
First Quartile | 2
Median | 2
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 7
Tables | 6
Tablets | 16
Replicas | 16
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240502 07:58:22.134284 16726 log_verifier.cc:126] Checking tablet 0cb1145d5f6343a1b205bb59221e95f4
I20240502 07:58:22.144527 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet 0cb1145d5f6343a1b205bb59221e95f4
I20240502 07:58:22.144596 16726 log_verifier.cc:126] Checking tablet 12852f6b0d7242208c3f2beea408bc89
I20240502 07:58:22.152268 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet 12852f6b0d7242208c3f2beea408bc89
I20240502 07:58:22.152312 16726 log_verifier.cc:126] Checking tablet 26330684e7454ddf983b85b66c99442e
I20240502 07:58:22.159689 16726 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 26330684e7454ddf983b85b66c99442e
I20240502 07:58:22.159739 16726 log_verifier.cc:126] Checking tablet 3b5a6f09d531470cbca30aa50654dcbf
I20240502 07:58:22.166111 16726 log_verifier.cc:177] Verified matching terms for 2 ops in tablet 3b5a6f09d531470cbca30aa50654dcbf
I20240502 07:58:22.166160 16726 log_verifier.cc:126] Checking tablet 4b9f9422ce124736b68e1933c95d4049
I20240502 07:58:22.173509 16726 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 4b9f9422ce124736b68e1933c95d4049
I20240502 07:58:22.173554 16726 log_verifier.cc:126] Checking tablet 52d9c7b102b24b48b81b8cd9642ecbdb
I20240502 07:58:22.180927 16726 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 52d9c7b102b24b48b81b8cd9642ecbdb
I20240502 07:58:22.180972 16726 log_verifier.cc:126] Checking tablet 53f83cc7a58447158d0dae01fe31adba
I20240502 07:58:22.188400 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet 53f83cc7a58447158d0dae01fe31adba
I20240502 07:58:22.188447 16726 log_verifier.cc:126] Checking tablet 63baa0ba36f742ca91fddceb0e4b9617
I20240502 07:58:22.195989 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet 63baa0ba36f742ca91fddceb0e4b9617
I20240502 07:58:22.196033 16726 log_verifier.cc:126] Checking tablet 708e377a224e40deab16180eb8f32c82
I20240502 07:58:22.203419 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet 708e377a224e40deab16180eb8f32c82
I20240502 07:58:22.203487 16726 log_verifier.cc:126] Checking tablet 72092a0eebee4b818fe377651dc77cbc
I20240502 07:58:22.210744 16726 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 72092a0eebee4b818fe377651dc77cbc
I20240502 07:58:22.210789 16726 log_verifier.cc:126] Checking tablet 823e3d11a041417e90344de4350fee72
I20240502 07:58:22.218183 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet 823e3d11a041417e90344de4350fee72
I20240502 07:58:22.218221 16726 log_verifier.cc:126] Checking tablet 95b1058e6dd5473ca41d534137ebb3eb
I20240502 07:58:22.225662 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet 95b1058e6dd5473ca41d534137ebb3eb
I20240502 07:58:22.225703 16726 log_verifier.cc:126] Checking tablet af6d015cb2154a29970c5a7a4116efe9
I20240502 07:58:22.234949 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet af6d015cb2154a29970c5a7a4116efe9
I20240502 07:58:22.234988 16726 log_verifier.cc:126] Checking tablet b2c2c3940c1f49c89cf0219dd481fa0b
I20240502 07:58:22.242568 16726 log_verifier.cc:177] Verified matching terms for 3 ops in tablet b2c2c3940c1f49c89cf0219dd481fa0b
I20240502 07:58:22.242609 16726 log_verifier.cc:126] Checking tablet ca8eef98ed8247d9b5a7086ec2b0f56b
I20240502 07:58:22.249534 16726 log_verifier.cc:177] Verified matching terms for 3 ops in tablet ca8eef98ed8247d9b5a7086ec2b0f56b
I20240502 07:58:22.249572 16726 log_verifier.cc:126] Checking tablet f8550e00f0ba4b5ea52822f99add2cfe
I20240502 07:58:22.256846 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet f8550e00f0ba4b5ea52822f99add2cfe
I20240502 07:58:22.258801 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.136:0
--local_ip_for_outbound_sockets=127.16.85.136
--webserver_interface=127.16.85.136
--webserver_port=0
--tserver_master_addrs=127.16.85.190:41469
--builtin_ntp_servers=127.16.85.148:33323
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 07:58:22.366744 21854 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:22.366923 21854 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:22.366961 21854 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:22.369655 21854 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:22.369709 21854 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:22.369762 21854 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.136
I20240502 07:58:22.373082 21854 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:33323
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.136:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.16.85.136
--webserver_port=0
--tserver_master_addrs=127.16.85.190:41469
--never_fsync=true
--heap_profile_path=/tmp/kudu.21854
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.136
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:22.373839 21854 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:22.374990 21854 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:22.386862 21859 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:22.387389 21862 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:22.388157 21860 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:22.388953 21854 server_base.cc:1034] running on GCE node
I20240502 07:58:22.389309 21854 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:22.389773 21854 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:22.390916 21854 hybrid_clock.cc:648] HybridClock initialized: now 1714636702390896 us; error 35 us; skew 500 ppm
I20240502 07:58:22.392536 21854 webserver.cc:460] Webserver started at http://127.16.85.136:35397/ using document root <none> and password file <none>
I20240502 07:58:22.393034 21854 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:22.393082 21854 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:22.393244 21854 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:22.394707 21854 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data/instance:
uuid: "1f5ab2991a0e42c9823de2f23c424200"
format_stamp: "Formatted at 2024-05-02 07:58:22 on dist-test-slave-7df7"
I20240502 07:58:22.395115 21854 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/instance:
uuid: "1f5ab2991a0e42c9823de2f23c424200"
format_stamp: "Formatted at 2024-05-02 07:58:22 on dist-test-slave-7df7"
I20240502 07:58:22.398058 21854 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 07:58:22.399821 21868 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:22.400715 21854 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
I20240502 07:58:22.400807 21854 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "1f5ab2991a0e42c9823de2f23c424200"
format_stamp: "Formatted at 2024-05-02 07:58:22 on dist-test-slave-7df7"
I20240502 07:58:22.400884 21854 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:22.420743 21854 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:22.421258 21854 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:22.421394 21854 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:22.421857 21854 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:22.422596 21854 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:22.422636 21854 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:22.422686 21854 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:22.422714 21854 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:22.429852 21854 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.136:35431
I20240502 07:58:22.429873 21981 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.136:35431 every 8 connection(s)
I20240502 07:58:22.430652 21854 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240502 07:58:22.436209 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 21854
I20240502 07:58:22.436311 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/instance
I20240502 07:58:22.438110 21982 heartbeater.cc:347] Connected to a master server at 127.16.85.190:41469
I20240502 07:58:22.438283 21982 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:22.438697 21982 heartbeater.cc:511] Master 127.16.85.190:41469 requested a full tablet report, sending...
I20240502 07:58:22.439507 18943 ts_manager.cc:194] Registered new tserver with Master: 1f5ab2991a0e42c9823de2f23c424200 (127.16.85.136:35431)
I20240502 07:58:22.439983 18943 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.136:56805
I20240502 07:58:22.601119 20329 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:22.601773 21916 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:22.603847 19198 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 07:58:22.610543 19066 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 07:58:22.622241 20591 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:22.628151 20198 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:22.633447 20713 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:22.641933 20460 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:22.724498 21916 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:22.730656 20198 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:22.730768 19066 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 07:58:22.732311 20460 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:22.734270 19198 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 07:58:22.734614 20591 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:22.749399 20713 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:22.754284 20329 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:22.786289 19087 consensus_queue.cc:237] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: true } }
I20240502 07:58:22.787083 22074 raft_consensus.cc:2932] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1f5ab2991a0e42c9823de2f23c424200 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: true } } }
I20240502 07:58:22.789510 18936 catalog_manager.cc:5484] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: config changed from index -1 to 4, NON_VOTER 1f5ab2991a0e42c9823de2f23c424200 (127.16.85.136) added. New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:22.790887 19022 consensus_peers.cc:485] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 -> Peer 1f5ab2991a0e42c9823de2f23c424200 (127.16.85.136:35431): Couldn't send request to peer 1f5ab2991a0e42c9823de2f23c424200. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ca8eef98ed8247d9b5a7086ec2b0f56b. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:22.795648 19218 consensus_queue.cc:237] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: true } }
I20240502 07:58:22.796362 22087 raft_consensus.cc:2932] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1f5ab2991a0e42c9823de2f23c424200 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: true } } }
I20240502 07:58:22.798396 18936 catalog_manager.cc:5484] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 reported cstate change: config changed from index -1 to 4, NON_VOTER 1f5ab2991a0e42c9823de2f23c424200 (127.16.85.136) added. New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:22.799712 19153 consensus_peers.cc:485] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 -> Peer 1f5ab2991a0e42c9823de2f23c424200 (127.16.85.136:35431): Couldn't send request to peer 1f5ab2991a0e42c9823de2f23c424200. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b2c2c3940c1f49c89cf0219dd481fa0b. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:23.379635 22121 ts_tablet_manager.cc:902] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200: Initiating tablet copy from peer 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130:33077)
I20240502 07:58:23.380220 22121 tablet_copy_client.cc:308] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33077
I20240502 07:58:23.383678 19238 tablet_copy_service.cc:140] P 3a68f86d2b9d44108fbb383fd0ec3118: Received BeginTabletCopySession request for tablet b2c2c3940c1f49c89cf0219dd481fa0b from peer 1f5ab2991a0e42c9823de2f23c424200 ({username='slave'} at 127.16.85.136:47777)
I20240502 07:58:23.383765 19238 tablet_copy_service.cc:161] P 3a68f86d2b9d44108fbb383fd0ec3118: Beginning new tablet copy session on tablet b2c2c3940c1f49c89cf0219dd481fa0b from peer 1f5ab2991a0e42c9823de2f23c424200 at {username='slave'} at 127.16.85.136:47777: session id = 1f5ab2991a0e42c9823de2f23c424200-b2c2c3940c1f49c89cf0219dd481fa0b
I20240502 07:58:23.384431 19238 tablet_copy_source_session.cc:205] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:23.387343 22121 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b2c2c3940c1f49c89cf0219dd481fa0b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:23.391947 22121 tablet_copy_client.cc:783] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:23.392107 22121 tablet_copy_client.cc:647] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:23.393500 22121 tablet_copy_client.cc:523] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:23.394933 22121 tablet_bootstrap.cc:492] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200: Bootstrap starting.
I20240502 07:58:23.395205 22128 ts_tablet_manager.cc:902] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200: Initiating tablet copy from peer dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153)
I20240502 07:58:23.395709 22128 tablet_copy_client.cc:308] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:44153
I20240502 07:58:23.398962 19107 tablet_copy_service.cc:140] P dfc05317d0c04450af55ceabe36f6a99: Received BeginTabletCopySession request for tablet ca8eef98ed8247d9b5a7086ec2b0f56b from peer 1f5ab2991a0e42c9823de2f23c424200 ({username='slave'} at 127.16.85.136:38493)
I20240502 07:58:23.399047 19107 tablet_copy_service.cc:161] P dfc05317d0c04450af55ceabe36f6a99: Beginning new tablet copy session on tablet ca8eef98ed8247d9b5a7086ec2b0f56b from peer 1f5ab2991a0e42c9823de2f23c424200 at {username='slave'} at 127.16.85.136:38493: session id = 1f5ab2991a0e42c9823de2f23c424200-ca8eef98ed8247d9b5a7086ec2b0f56b
I20240502 07:58:23.399645 19107 tablet_copy_source_session.cc:205] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:23.400312 22128 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ca8eef98ed8247d9b5a7086ec2b0f56b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:23.400437 22121 log.cc:824] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:23.401937 22128 tablet_copy_client.cc:783] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:23.402060 22128 tablet_copy_client.cc:647] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:23.403036 22128 tablet_copy_client.cc:523] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:23.403824 22128 tablet_bootstrap.cc:492] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200: Bootstrap starting.
I20240502 07:58:23.417052 22121 tablet_bootstrap.cc:492] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:23.417268 22121 tablet_bootstrap.cc:492] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200: Bootstrap complete.
I20240502 07:58:23.417477 22121 ts_tablet_manager.cc:1367] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200: Time spent bootstrapping tablet: real 0.023s user 0.014s sys 0.009s
I20240502 07:58:23.418764 22128 tablet_bootstrap.cc:492] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:23.418951 22128 tablet_bootstrap.cc:492] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200: Bootstrap complete.
I20240502 07:58:23.418958 22121 raft_consensus.cc:348] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: true } }
I20240502 07:58:23.419102 22128 ts_tablet_manager.cc:1367] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200: Time spent bootstrapping tablet: real 0.015s user 0.011s sys 0.005s
I20240502 07:58:23.419154 22121 raft_consensus.cc:729] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1f5ab2991a0e42c9823de2f23c424200, State: Initialized, Role: LEARNER
I20240502 07:58:23.419416 22121 consensus_queue.cc:260] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: true } }
I20240502 07:58:23.419447 22128 raft_consensus.cc:348] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: true } }
I20240502 07:58:23.419525 22128 raft_consensus.cc:729] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1f5ab2991a0e42c9823de2f23c424200, State: Initialized, Role: LEARNER
I20240502 07:58:23.419602 22128 consensus_queue.cc:260] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: true } }
I20240502 07:58:23.419912 21982 heartbeater.cc:503] Master 127.16.85.190:41469 was elected leader, sending a full tablet report...
I20240502 07:58:23.420037 22121 ts_tablet_manager.cc:1398] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200: Time spent starting tablet: real 0.003s user 0.002s sys 0.001s
I20240502 07:58:23.420181 22128 ts_tablet_manager.cc:1398] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:23.420578 19107 tablet_copy_service.cc:342] P dfc05317d0c04450af55ceabe36f6a99: Request end of tablet copy session 1f5ab2991a0e42c9823de2f23c424200-ca8eef98ed8247d9b5a7086ec2b0f56b received from {username='slave'} at 127.16.85.136:38493
I20240502 07:58:23.420688 19107 tablet_copy_service.cc:433] P dfc05317d0c04450af55ceabe36f6a99: ending tablet copy session 1f5ab2991a0e42c9823de2f23c424200-ca8eef98ed8247d9b5a7086ec2b0f56b on tablet ca8eef98ed8247d9b5a7086ec2b0f56b with peer 1f5ab2991a0e42c9823de2f23c424200
I20240502 07:58:23.420971 19238 tablet_copy_service.cc:342] P 3a68f86d2b9d44108fbb383fd0ec3118: Request end of tablet copy session 1f5ab2991a0e42c9823de2f23c424200-b2c2c3940c1f49c89cf0219dd481fa0b received from {username='slave'} at 127.16.85.136:47777
I20240502 07:58:23.421056 19238 tablet_copy_service.cc:433] P 3a68f86d2b9d44108fbb383fd0ec3118: ending tablet copy session 1f5ab2991a0e42c9823de2f23c424200-b2c2c3940c1f49c89cf0219dd481fa0b on tablet b2c2c3940c1f49c89cf0219dd481fa0b with peer 1f5ab2991a0e42c9823de2f23c424200
I20240502 07:58:23.789692 21936 raft_consensus.cc:1203] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:23.867327 21936 raft_consensus.cc:1203] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:24.323980 22191 raft_consensus.cc:1050] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118: attempting to promote NON_VOTER 1f5ab2991a0e42c9823de2f23c424200 to VOTER
I20240502 07:58:24.324260 22191 consensus_queue.cc:237] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } }
I20240502 07:58:24.325225 21936 raft_consensus.cc:1261] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [term 1 LEARNER]: Refusing update from remote peer 3a68f86d2b9d44108fbb383fd0ec3118: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:24.325556 22191 consensus_queue.cc:1035] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:24.327580 22192 raft_consensus.cc:2932] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1f5ab2991a0e42c9823de2f23c424200 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } } }
I20240502 07:58:24.327991 21936 raft_consensus.cc:2932] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1f5ab2991a0e42c9823de2f23c424200 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } } }
I20240502 07:58:24.329749 18941 catalog_manager.cc:5484] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 reported cstate change: config changed from index 4 to 5, 1f5ab2991a0e42c9823de2f23c424200 (127.16.85.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:24.410724 22200 raft_consensus.cc:1050] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99: attempting to promote NON_VOTER 1f5ab2991a0e42c9823de2f23c424200 to VOTER
I20240502 07:58:24.411062 22200 consensus_queue.cc:237] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } }
I20240502 07:58:24.411985 21936 raft_consensus.cc:1261] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [term 1 LEARNER]: Refusing update from remote peer dfc05317d0c04450af55ceabe36f6a99: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:24.412293 22200 consensus_queue.cc:1035] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:24.413631 22201 raft_consensus.cc:2932] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1f5ab2991a0e42c9823de2f23c424200 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } } }
I20240502 07:58:24.413923 21936 raft_consensus.cc:2932] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1f5ab2991a0e42c9823de2f23c424200 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } } }
I20240502 07:58:24.415542 18943 catalog_manager.cc:5484] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 reported cstate change: config changed from index 4 to 5, 1f5ab2991a0e42c9823de2f23c424200 (127.16.85.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dfc05317d0c04450af55ceabe36f6a99" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:24.485728 19218 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "b2c2c3940c1f49c89cf0219dd481fa0b"
dest_uuid: "3a68f86d2b9d44108fbb383fd0ec3118"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:55118
I20240502 07:58:24.485834 19218 raft_consensus.cc:595] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:24.493986 19087 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "ca8eef98ed8247d9b5a7086ec2b0f56b"
dest_uuid: "dfc05317d0c04450af55ceabe36f6a99"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40894
I20240502 07:58:24.494082 19087 raft_consensus.cc:595] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:24.702903 19218 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "b2c2c3940c1f49c89cf0219dd481fa0b"
dest_uuid: "3a68f86d2b9d44108fbb383fd0ec3118"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:55130
I20240502 07:58:24.703022 19218 raft_consensus.cc:595] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:24.710898 19087 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "ca8eef98ed8247d9b5a7086ec2b0f56b"
dest_uuid: "dfc05317d0c04450af55ceabe36f6a99"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40922
I20240502 07:58:24.711020 19087 raft_consensus.cc:595] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:24.804076 22201 raft_consensus.cc:979] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99: : Instructing follower 1f5ab2991a0e42c9823de2f23c424200 to start an election
I20240502 07:58:24.804196 22201 raft_consensus.cc:1067] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Signalling peer 1f5ab2991a0e42c9823de2f23c424200 to start an election
I20240502 07:58:24.804628 21936 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "ca8eef98ed8247d9b5a7086ec2b0f56b"
dest_uuid: "1f5ab2991a0e42c9823de2f23c424200"
from {username='slave'} at 127.16.85.129:52187
I20240502 07:58:24.804850 21936 raft_consensus.cc:482] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 07:58:24.804972 21936 raft_consensus.cc:3037] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:24.805788 21936 raft_consensus.cc:504] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } }
I20240502 07:58:24.806677 21936 leader_election.cc:288] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [CANDIDATE]: Term 2 election: Requested vote from peers dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153)
I20240502 07:58:24.807006 19087 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca8eef98ed8247d9b5a7086ec2b0f56b" candidate_uuid: "1f5ab2991a0e42c9823de2f23c424200" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "dfc05317d0c04450af55ceabe36f6a99"
I20240502 07:58:24.807122 19087 raft_consensus.cc:3032] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 07:58:24.807181 19087 raft_consensus.cc:729] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: dfc05317d0c04450af55ceabe36f6a99, State: Running, Role: LEADER
I20240502 07:58:24.807332 19087 consensus_queue.cc:260] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } }
I20240502 07:58:24.807474 19087 raft_consensus.cc:3037] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:24.808223 19087 raft_consensus.cc:2454] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1f5ab2991a0e42c9823de2f23c424200 in term 2.
I20240502 07:58:24.808542 21871 leader_election.cc:302] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 1f5ab2991a0e42c9823de2f23c424200, dfc05317d0c04450af55ceabe36f6a99; no voters:
I20240502 07:58:24.808753 22135 raft_consensus.cc:2781] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:24.808912 22135 raft_consensus.cc:686] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [term 2 LEADER]: Becoming Leader. State: Replica: 1f5ab2991a0e42c9823de2f23c424200, State: Running, Role: LEADER
I20240502 07:58:24.809165 22135 consensus_queue.cc:237] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } }
I20240502 07:58:24.810927 18941 catalog_manager.cc:5484] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 reported cstate change: term changed from 1 to 2, leader changed from dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) to 1f5ab2991a0e42c9823de2f23c424200 (127.16.85.136). New cstate: current_term: 2 leader_uuid: "1f5ab2991a0e42c9823de2f23c424200" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
W20240502 07:58:24.816008 18931 catalog_manager.cc:5014] ChangeConfig:REMOVE_PEER RPC for tablet ca8eef98ed8247d9b5a7086ec2b0f56b with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER failed with leader 1f5ab2991a0e42c9823de2f23c424200 (127.16.85.136:35431) due to error UNKNOWN_ERROR; will retry: Illegal state: Leader has not yet committed an operation in its own term
I20240502 07:58:24.879920 22248 raft_consensus.cc:979] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118: : Instructing follower 1f5ab2991a0e42c9823de2f23c424200 to start an election
I20240502 07:58:24.880009 22248 raft_consensus.cc:1067] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Signalling peer 1f5ab2991a0e42c9823de2f23c424200 to start an election
I20240502 07:58:24.880391 21936 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "b2c2c3940c1f49c89cf0219dd481fa0b"
dest_uuid: "1f5ab2991a0e42c9823de2f23c424200"
from {username='slave'} at 127.16.85.130:52609
I20240502 07:58:24.880517 21936 raft_consensus.cc:482] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 07:58:24.880578 21936 raft_consensus.cc:3037] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:24.881400 21936 raft_consensus.cc:504] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } }
I20240502 07:58:24.881752 21936 leader_election.cc:288] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [CANDIDATE]: Term 2 election: Requested vote from peers 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130:33077)
I20240502 07:58:24.882109 19218 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b2c2c3940c1f49c89cf0219dd481fa0b" candidate_uuid: "1f5ab2991a0e42c9823de2f23c424200" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "3a68f86d2b9d44108fbb383fd0ec3118"
I20240502 07:58:24.882202 19218 raft_consensus.cc:3032] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 07:58:24.882230 19218 raft_consensus.cc:729] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 3a68f86d2b9d44108fbb383fd0ec3118, State: Running, Role: LEADER
I20240502 07:58:24.882339 19218 consensus_queue.cc:260] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } }
I20240502 07:58:24.882439 19218 raft_consensus.cc:3037] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:24.883035 19218 raft_consensus.cc:2454] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1f5ab2991a0e42c9823de2f23c424200 in term 2.
I20240502 07:58:24.883280 21872 leader_election.cc:302] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 1f5ab2991a0e42c9823de2f23c424200, 3a68f86d2b9d44108fbb383fd0ec3118; no voters:
I20240502 07:58:24.883422 22135 raft_consensus.cc:2781] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:24.883503 22135 raft_consensus.cc:686] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [term 2 LEADER]: Becoming Leader. State: Replica: 1f5ab2991a0e42c9823de2f23c424200, State: Running, Role: LEADER
I20240502 07:58:24.883637 22135 consensus_queue.cc:237] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } }
I20240502 07:58:24.884900 18941 catalog_manager.cc:5484] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 reported cstate change: term changed from 1 to 2, leader changed from 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130) to 1f5ab2991a0e42c9823de2f23c424200 (127.16.85.136). New cstate: current_term: 2 leader_uuid: "1f5ab2991a0e42c9823de2f23c424200" committed_config { opid_index: 5 OBSOLETE_local: true peers { permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:25.266430 19087 raft_consensus.cc:1261] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Refusing update from remote peer 1f5ab2991a0e42c9823de2f23c424200: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 07:58:25.266932 22135 consensus_queue.cc:1035] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dfc05317d0c04450af55ceabe36f6a99" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 44153 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:25.332604 19218 raft_consensus.cc:1261] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Refusing update from remote peer 1f5ab2991a0e42c9823de2f23c424200: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 07:58:25.332993 22135 consensus_queue.cc:1035] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3a68f86d2b9d44108fbb383fd0ec3118" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33077 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:25.451354 21936 consensus_queue.cc:237] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } }
I20240502 07:58:25.451926 22245 raft_consensus.cc:2932] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) evicted. New config: { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } } }
I20240502 07:58:25.453068 18931 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ca8eef98ed8247d9b5a7086ec2b0f56b with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 07:58:25.453862 18941 catalog_manager.cc:5484] T ca8eef98ed8247d9b5a7086ec2b0f56b P 1f5ab2991a0e42c9823de2f23c424200 reported cstate change: config changed from index 5 to 7, VOTER dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129) evicted. New cstate: current_term: 2 leader_uuid: "1f5ab2991a0e42c9823de2f23c424200" committed_config { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:25.456203 19066 tablet_service.cc:1514] Processing DeleteTablet for tablet ca8eef98ed8247d9b5a7086ec2b0f56b with delete_type TABLET_DATA_TOMBSTONED (TS dfc05317d0c04450af55ceabe36f6a99 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:36952
I20240502 07:58:25.456480 22287 tablet_replica.cc:330] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99: stopping tablet replica
I20240502 07:58:25.456600 22287 raft_consensus.cc:2229] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:25.456718 22287 raft_consensus.cc:2258] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:25.457855 22287 ts_tablet_manager.cc:1875] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:25.459612 22287 ts_tablet_manager.cc:1888] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 07:58:25.459687 22287 log.cc:1196] T ca8eef98ed8247d9b5a7086ec2b0f56b P dfc05317d0c04450af55ceabe36f6a99: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/ca8eef98ed8247d9b5a7086ec2b0f56b
I20240502 07:58:25.460105 18931 catalog_manager.cc:4830] TS dfc05317d0c04450af55ceabe36f6a99 (127.16.85.129:44153): tablet ca8eef98ed8247d9b5a7086ec2b0f56b (table rebalance_test_table_4 [id=db78e3e5980d48a3886a4b5403848cb3]) successfully deleted
I20240502 07:58:25.503085 21936 consensus_queue.cc:237] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } }
I20240502 07:58:25.503654 22245 raft_consensus.cc:2932] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130) evicted. New config: { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } } }
I20240502 07:58:25.504683 18931 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b2c2c3940c1f49c89cf0219dd481fa0b with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 6)
I20240502 07:58:25.505352 18941 catalog_manager.cc:5484] T b2c2c3940c1f49c89cf0219dd481fa0b P 1f5ab2991a0e42c9823de2f23c424200 reported cstate change: config changed from index 5 to 7, VOTER 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130) evicted. New cstate: current_term: 2 leader_uuid: "1f5ab2991a0e42c9823de2f23c424200" committed_config { opid_index: 7 OBSOLETE_local: true peers { permanent_uuid: "1f5ab2991a0e42c9823de2f23c424200" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 35431 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:25.507155 19198 tablet_service.cc:1514] Processing DeleteTablet for tablet b2c2c3940c1f49c89cf0219dd481fa0b with delete_type TABLET_DATA_TOMBSTONED (TS 3a68f86d2b9d44108fbb383fd0ec3118 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:59140
I20240502 07:58:25.507452 22288 tablet_replica.cc:330] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118: stopping tablet replica
I20240502 07:58:25.507546 22288 raft_consensus.cc:2229] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:25.507627 22288 raft_consensus.cc:2258] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:25.508801 22288 ts_tablet_manager.cc:1875] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:25.510329 22288 ts_tablet_manager.cc:1888] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 07:58:25.510386 22288 log.cc:1196] T b2c2c3940c1f49c89cf0219dd481fa0b P 3a68f86d2b9d44108fbb383fd0ec3118: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/b2c2c3940c1f49c89cf0219dd481fa0b
I20240502 07:58:25.510773 18932 catalog_manager.cc:4830] TS 3a68f86d2b9d44108fbb383fd0ec3118 (127.16.85.130:33077): tablet b2c2c3940c1f49c89cf0219dd481fa0b (table rebalance_test_table_3 [id=44e95bee069c42d2b54ac8810ce8424f]) successfully deleted
I20240502 07:58:25.599907 21916 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.600324 20460 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.601303 20329 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.609134 20713 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.621460 19066 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.626325 20198 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.638075 20591 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.645077 19198 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.714769 21916 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.718006 20329 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.720268 20198 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.720314 20591 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.739609 19066 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.740866 19198 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.741655 20460 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.744110 20713 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.821167 20329 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.824410 21916 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.825744 19198 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.826340 20460 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.830327 20713 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.837852 19066 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.838444 20198 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:25.841572 20591 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
2229511e8632482d8a20ec64c9c6ea56 | 127.16.85.190:41469 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
allow_unsafe_replication_factor | true | runtime,unsafe | all 1 server(s) checked
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.16.85.148:33323 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+---------+----------+----------------+-----------------
1f5ab2991a0e42c9823de2f23c424200 | 127.16.85.136:35431 | HEALTHY | <none> | 2 | 0
278943ee039c48988108fc425b366ff7 | 127.16.85.132:44867 | HEALTHY | <none> | 2 | 0
3a4ce39a207341a4883e10e934b839b4 | 127.16.85.133:36251 | HEALTHY | <none> | 2 | 0
3a68f86d2b9d44108fbb383fd0ec3118 | 127.16.85.130:33077 | HEALTHY | <none> | 2 | 0
74506b6a243b450398cf31bdabf3eccf | 127.16.85.134:36697 | HEALTHY | <none> | 2 | 0
cbc178a0a2f545658c5961645415eb25 | 127.16.85.131:42907 | HEALTHY | <none> | 2 | 0
dfc05317d0c04450af55ceabe36f6a99 | 127.16.85.129:44153 | HEALTHY | <none> | 2 | 0
f81ce2156e854e0ab96f4b6fb2077714 | 127.16.85.135:45589 | HEALTHY | <none> | 2 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 8
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 8 server(s) checked
local_ip_for_outbound_sockets | 127.16.85.129 | experimental | 127.16.85.129:44153
local_ip_for_outbound_sockets | 127.16.85.130 | experimental | 127.16.85.130:33077
local_ip_for_outbound_sockets | 127.16.85.131 | experimental | 127.16.85.131:42907
local_ip_for_outbound_sockets | 127.16.85.132 | experimental | 127.16.85.132:44867
local_ip_for_outbound_sockets | 127.16.85.133 | experimental | 127.16.85.133:36251
local_ip_for_outbound_sockets | 127.16.85.134 | experimental | 127.16.85.134:36697
local_ip_for_outbound_sockets | 127.16.85.135 | experimental | 127.16.85.135:45589
local_ip_for_outbound_sockets | 127.16.85.136 | experimental | 127.16.85.136:35431
never_fsync | true | unsafe,advanced | all 8 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 8 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 8 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 8 server(s) checked
server_dump_info_format | pb | hidden | all 8 server(s) checked
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.16.85.129:44153
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.16.85.130:33077
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.16.85.131:42907
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.16.85.132:44867
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.16.85.133:36251
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.16.85.134:36697
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/info.pb | hidden | 127.16.85.135:45589
server_dump_info_path | /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.RebalanceParamTest.Rebalance_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data/info.pb | hidden | 127.16.85.136:35431
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.16.85.148:33323 | all 8 server(s) checked
time_source | builtin | all 8 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 9 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
TestTable | 1 | HEALTHY | 1 | 1 | 0 | 0 | 0
rebalance_test_table_0 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_4 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 2
First Quartile | 2
Median | 2
Third Quartile | 2
Maximum | 2
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 8
Tables | 6
Tablets | 16
Replicas | 16
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240502 07:58:25.869263 16726 log_verifier.cc:126] Checking tablet 0cb1145d5f6343a1b205bb59221e95f4
I20240502 07:58:25.874990 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet 0cb1145d5f6343a1b205bb59221e95f4
I20240502 07:58:25.875030 16726 log_verifier.cc:126] Checking tablet 12852f6b0d7242208c3f2beea408bc89
I20240502 07:58:25.879966 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet 12852f6b0d7242208c3f2beea408bc89
I20240502 07:58:25.880002 16726 log_verifier.cc:126] Checking tablet 26330684e7454ddf983b85b66c99442e
I20240502 07:58:25.884850 16726 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 26330684e7454ddf983b85b66c99442e
I20240502 07:58:25.884882 16726 log_verifier.cc:126] Checking tablet 3b5a6f09d531470cbca30aa50654dcbf
I20240502 07:58:25.888564 16726 log_verifier.cc:177] Verified matching terms for 2 ops in tablet 3b5a6f09d531470cbca30aa50654dcbf
I20240502 07:58:25.888597 16726 log_verifier.cc:126] Checking tablet 4b9f9422ce124736b68e1933c95d4049
I20240502 07:58:25.893404 16726 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 4b9f9422ce124736b68e1933c95d4049
I20240502 07:58:25.893435 16726 log_verifier.cc:126] Checking tablet 52d9c7b102b24b48b81b8cd9642ecbdb
I20240502 07:58:25.898250 16726 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 52d9c7b102b24b48b81b8cd9642ecbdb
I20240502 07:58:25.898296 16726 log_verifier.cc:126] Checking tablet 53f83cc7a58447158d0dae01fe31adba
I20240502 07:58:25.903245 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet 53f83cc7a58447158d0dae01fe31adba
I20240502 07:58:25.903285 16726 log_verifier.cc:126] Checking tablet 63baa0ba36f742ca91fddceb0e4b9617
I20240502 07:58:25.908028 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet 63baa0ba36f742ca91fddceb0e4b9617
I20240502 07:58:25.908063 16726 log_verifier.cc:126] Checking tablet 708e377a224e40deab16180eb8f32c82
I20240502 07:58:25.913250 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet 708e377a224e40deab16180eb8f32c82
I20240502 07:58:25.913285 16726 log_verifier.cc:126] Checking tablet 72092a0eebee4b818fe377651dc77cbc
I20240502 07:58:25.917994 16726 log_verifier.cc:177] Verified matching terms for 3 ops in tablet 72092a0eebee4b818fe377651dc77cbc
I20240502 07:58:25.918027 16726 log_verifier.cc:126] Checking tablet 823e3d11a041417e90344de4350fee72
I20240502 07:58:25.922981 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet 823e3d11a041417e90344de4350fee72
I20240502 07:58:25.923014 16726 log_verifier.cc:126] Checking tablet 95b1058e6dd5473ca41d534137ebb3eb
I20240502 07:58:25.927699 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet 95b1058e6dd5473ca41d534137ebb3eb
I20240502 07:58:25.927732 16726 log_verifier.cc:126] Checking tablet af6d015cb2154a29970c5a7a4116efe9
I20240502 07:58:25.932874 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet af6d015cb2154a29970c5a7a4116efe9
I20240502 07:58:25.932906 16726 log_verifier.cc:126] Checking tablet b2c2c3940c1f49c89cf0219dd481fa0b
I20240502 07:58:25.939914 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet b2c2c3940c1f49c89cf0219dd481fa0b
I20240502 07:58:25.939955 16726 log_verifier.cc:126] Checking tablet ca8eef98ed8247d9b5a7086ec2b0f56b
I20240502 07:58:25.947445 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet ca8eef98ed8247d9b5a7086ec2b0f56b
I20240502 07:58:25.947484 16726 log_verifier.cc:126] Checking tablet f8550e00f0ba4b5ea52822f99add2cfe
I20240502 07:58:25.952745 16726 log_verifier.cc:177] Verified matching terms for 7 ops in tablet f8550e00f0ba4b5ea52822f99add2cfe
I20240502 07:58:25.953115 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 19005
I20240502 07:58:25.959535 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 19136
I20240502 07:58:25.965004 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 20136
I20240502 07:58:25.969871 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 20267
I20240502 07:58:25.974961 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 20398
I20240502 07:58:25.979872 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 20529
I20240502 07:58:25.984809 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 20660
I20240502 07:58:25.989517 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 21854
I20240502 07:58:25.994477 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 18914
2024-05-02T07:58:25Z chronyd exiting
[ OK ] RebalanceParamTest.Rebalance/1 (14935 ms)
[----------] 1 test from RebalanceParamTest (14935 ms total)
[----------] 1 test from DDLDuringRebalancingTest
[ RUN ] DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing/1
I20240502 07:58:26.029547 16726 test_util.cc:268] Using random seed: -137879578
I20240502 07:58:26.030210 16726 ts_itest-base.cc:112] Starting cluster with:
I20240502 07:58:26.030242 16726 ts_itest-base.cc:113] --------------
I20240502 07:58:26.030252 16726 ts_itest-base.cc:114] 8 tablet servers
I20240502 07:58:26.030262 16726 ts_itest-base.cc:115] 3 replicas per TS
I20240502 07:58:26.030282 16726 ts_itest-base.cc:116] --------------
2024-05-02T07:58:26Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T07:58:26Z Disabled control of system clock
I20240502 07:58:26.037421 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.16.85.190:39083
--webserver_interface=127.16.85.190
--webserver_port=0
--builtin_ntp_servers=127.16.85.148:41971
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.16.85.190:39083
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false with env {}
W20240502 07:58:26.130513 22413 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:26.130707 22413 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:26.130759 22413 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:26.133486 22413 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:26.133534 22413 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 07:58:26.133553 22413 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:26.133569 22413 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 07:58:26.133582 22413 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 07:58:26.136516 22413 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:41971
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.16.85.190:39083
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.16.85.190:39083
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.16.85.190
--webserver_port=0
--never_fsync=true
--heap_profile_path=/tmp/kudu.22413
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:26.137233 22413 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:26.138203 22413 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:26.145179 22421 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:26.145656 22418 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:26.146322 22413 server_base.cc:1034] running on GCE node
W20240502 07:58:26.147171 22419 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:26.147588 22413 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:26.148377 22413 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:26.149503 22413 hybrid_clock.cc:648] HybridClock initialized: now 1714636706149492 us; error 25 us; skew 500 ppm
I20240502 07:58:26.151197 22413 webserver.cc:460] Webserver started at http://127.16.85.190:39759/ using document root <none> and password file <none>
I20240502 07:58:26.151696 22413 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:26.151749 22413 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:26.151923 22413 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:26.153429 22413 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "a69026f7179945c0b6a4b19bfb88d16e"
format_stamp: "Formatted at 2024-05-02 07:58:26 on dist-test-slave-7df7"
I20240502 07:58:26.153832 22413 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "a69026f7179945c0b6a4b19bfb88d16e"
format_stamp: "Formatted at 2024-05-02 07:58:26 on dist-test-slave-7df7"
I20240502 07:58:26.156533 22413 fs_manager.cc:691] Time spent creating directory manager: real 0.002s user 0.003s sys 0.000s
I20240502 07:58:26.158337 22427 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:26.159155 22413 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:26.159243 22413 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
uuid: "a69026f7179945c0b6a4b19bfb88d16e"
format_stamp: "Formatted at 2024-05-02 07:58:26 on dist-test-slave-7df7"
I20240502 07:58:26.159308 22413 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:26.187992 22413 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:26.188558 22413 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:26.188721 22413 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:26.194397 22413 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.190:39083
I20240502 07:58:26.194407 22479 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.190:39083 every 8 connection(s)
I20240502 07:58:26.195220 22413 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 07:58:26.197479 22480 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:26.201543 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 22413
I20240502 07:58:26.201643 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 07:58:26.201908 22480 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Bootstrap starting.
I20240502 07:58:26.203953 22480 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:26.204511 22480 log.cc:824] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:26.205796 22480 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: No bootstrap required, opened a new log
I20240502 07:58:26.208149 22480 raft_consensus.cc:348] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a69026f7179945c0b6a4b19bfb88d16e" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 39083 } }
I20240502 07:58:26.208326 22480 raft_consensus.cc:374] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:26.208384 22480 raft_consensus.cc:729] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a69026f7179945c0b6a4b19bfb88d16e, State: Initialized, Role: FOLLOWER
I20240502 07:58:26.208822 22480 consensus_queue.cc:260] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a69026f7179945c0b6a4b19bfb88d16e" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 39083 } }
I20240502 07:58:26.208930 22480 raft_consensus.cc:388] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 07:58:26.208985 22480 raft_consensus.cc:482] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 07:58:26.209086 22480 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:26.209743 22480 raft_consensus.cc:504] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a69026f7179945c0b6a4b19bfb88d16e" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 39083 } }
I20240502 07:58:26.210016 22480 leader_election.cc:302] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: a69026f7179945c0b6a4b19bfb88d16e; no voters:
I20240502 07:58:26.210222 22480 leader_election.cc:288] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 07:58:26.210525 22485 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:26.210690 22485 raft_consensus.cc:686] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [term 1 LEADER]: Becoming Leader. State: Replica: a69026f7179945c0b6a4b19bfb88d16e, State: Running, Role: LEADER
I20240502 07:58:26.210903 22485 consensus_queue.cc:237] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a69026f7179945c0b6a4b19bfb88d16e" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 39083 } }
I20240502 07:58:26.211158 22480 sys_catalog.cc:564] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [sys.catalog]: configured and running, proceeding with master startup.
I20240502 07:58:26.212276 22486 sys_catalog.cc:455] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "a69026f7179945c0b6a4b19bfb88d16e" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a69026f7179945c0b6a4b19bfb88d16e" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 39083 } } }
I20240502 07:58:26.212276 22487 sys_catalog.cc:455] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [sys.catalog]: SysCatalogTable state changed. Reason: New leader a69026f7179945c0b6a4b19bfb88d16e. Latest consensus state: current_term: 1 leader_uuid: "a69026f7179945c0b6a4b19bfb88d16e" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a69026f7179945c0b6a4b19bfb88d16e" member_type: VOTER last_known_addr { host: "127.16.85.190" port: 39083 } } }
I20240502 07:58:26.212400 22486 sys_catalog.cc:458] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [sys.catalog]: This master's current role is: LEADER
I20240502 07:58:26.212448 22487 sys_catalog.cc:458] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e [sys.catalog]: This master's current role is: LEADER
I20240502 07:58:26.212766 22494 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 07:58:26.215240 22494 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 07:58:26.219605 22494 catalog_manager.cc:1347] Generated new cluster ID: 0a8aa39f01034807be4b7a68ee3be74e
I20240502 07:58:26.219666 22494 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 07:58:26.231930 22494 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 07:58:26.232724 22494 catalog_manager.cc:1504] Loading token signing keys...
I20240502 07:58:26.239702 22494 catalog_manager.cc:5857] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Generated new TSK 0
I20240502 07:58:26.240272 22494 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 07:58:26.252386 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.129:0
--local_ip_for_outbound_sockets=127.16.85.129
--webserver_interface=127.16.85.129
--webserver_port=0
--tserver_master_addrs=127.16.85.190:39083
--builtin_ntp_servers=127.16.85.148:41971
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 07:58:26.347393 22504 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:26.347560 22504 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:26.347596 22504 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:26.350129 22504 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:26.350209 22504 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:26.350302 22504 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.129
I20240502 07:58:26.353487 22504 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:41971
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.16.85.129
--webserver_port=0
--tserver_master_addrs=127.16.85.190:39083
--never_fsync=true
--heap_profile_path=/tmp/kudu.22504
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.129
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:26.354178 22504 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:26.355104 22504 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:26.362874 22512 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:26.363626 22509 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:26.364269 22510 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:26.364956 22504 server_base.cc:1034] running on GCE node
I20240502 07:58:26.365335 22504 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:26.365828 22504 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:26.366962 22504 hybrid_clock.cc:648] HybridClock initialized: now 1714636706366950 us; error 27 us; skew 500 ppm
I20240502 07:58:26.368569 22504 webserver.cc:460] Webserver started at http://127.16.85.129:35989/ using document root <none> and password file <none>
I20240502 07:58:26.369058 22504 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:26.369110 22504 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:26.369253 22504 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:26.370672 22504 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
format_stamp: "Formatted at 2024-05-02 07:58:26 on dist-test-slave-7df7"
I20240502 07:58:26.371104 22504 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
format_stamp: "Formatted at 2024-05-02 07:58:26 on dist-test-slave-7df7"
I20240502 07:58:26.374053 22504 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 07:58:26.375871 22518 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:26.376756 22504 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
I20240502 07:58:26.376844 22504 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
format_stamp: "Formatted at 2024-05-02 07:58:26 on dist-test-slave-7df7"
I20240502 07:58:26.376914 22504 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:26.394852 22504 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:26.395357 22504 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:26.395481 22504 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:26.395893 22504 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:26.396633 22504 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:26.396704 22504 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:26.396750 22504 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:26.396775 22504 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:26.403653 22504 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.129:37451
I20240502 07:58:26.403683 22631 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.129:37451 every 8 connection(s)
I20240502 07:58:26.404453 22504 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 07:58:26.407110 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 22504
I20240502 07:58:26.407227 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 07:58:26.409123 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.130:0
--local_ip_for_outbound_sockets=127.16.85.130
--webserver_interface=127.16.85.130
--webserver_port=0
--tserver_master_addrs=127.16.85.190:39083
--builtin_ntp_servers=127.16.85.148:41971
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:26.412200 22632 heartbeater.cc:347] Connected to a master server at 127.16.85.190:39083
I20240502 07:58:26.412405 22632 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:26.412875 22632 heartbeater.cc:511] Master 127.16.85.190:39083 requested a full tablet report, sending...
I20240502 07:58:26.414080 22444 ts_manager.cc:194] Registered new tserver with Master: 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:26.415028 22444 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.129:55781
W20240502 07:58:26.500147 22635 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:26.500308 22635 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:26.500370 22635 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:26.502997 22635 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:26.503065 22635 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:26.503137 22635 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.130
I20240502 07:58:26.506089 22635 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:41971
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.16.85.130
--webserver_port=0
--tserver_master_addrs=127.16.85.190:39083
--never_fsync=true
--heap_profile_path=/tmp/kudu.22635
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.130
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:26.506693 22635 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:26.507520 22635 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:26.514535 22643 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:26.515267 22640 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:26.515973 22635 server_base.cc:1034] running on GCE node
W20240502 07:58:26.516412 22641 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:26.516736 22635 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:26.517167 22635 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:26.518304 22635 hybrid_clock.cc:648] HybridClock initialized: now 1714636706518294 us; error 22 us; skew 500 ppm
I20240502 07:58:26.519735 22635 webserver.cc:460] Webserver started at http://127.16.85.130:33803/ using document root <none> and password file <none>
I20240502 07:58:26.520154 22635 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:26.520200 22635 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:26.520339 22635 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:26.521893 22635 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "725969d490a04fdfb3ba890c3f5096e3"
format_stamp: "Formatted at 2024-05-02 07:58:26 on dist-test-slave-7df7"
I20240502 07:58:26.522437 22635 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "725969d490a04fdfb3ba890c3f5096e3"
format_stamp: "Formatted at 2024-05-02 07:58:26 on dist-test-slave-7df7"
I20240502 07:58:26.525331 22635 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.004s sys 0.000s
I20240502 07:58:26.526937 22649 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:26.527740 22635 fs_manager.cc:725] Time spent opening block manager: real 0.001s user 0.002s sys 0.000s
I20240502 07:58:26.527818 22635 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "725969d490a04fdfb3ba890c3f5096e3"
format_stamp: "Formatted at 2024-05-02 07:58:26 on dist-test-slave-7df7"
I20240502 07:58:26.527877 22635 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:26.550732 22635 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:26.551154 22635 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:26.551266 22635 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:26.551648 22635 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:26.552364 22635 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:26.552402 22635 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:26.552433 22635 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:26.552450 22635 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:26.559537 22635 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.130:33387
I20240502 07:58:26.559562 22762 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.130:33387 every 8 connection(s)
I20240502 07:58:26.560367 22635 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 07:58:26.563956 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 22635
I20240502 07:58:26.564054 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 07:58:26.566444 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.131:0
--local_ip_for_outbound_sockets=127.16.85.131
--webserver_interface=127.16.85.131
--webserver_port=0
--tserver_master_addrs=127.16.85.190:39083
--builtin_ntp_servers=127.16.85.148:41971
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:26.567909 22763 heartbeater.cc:347] Connected to a master server at 127.16.85.190:39083
I20240502 07:58:26.568092 22763 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:26.568511 22763 heartbeater.cc:511] Master 127.16.85.190:39083 requested a full tablet report, sending...
I20240502 07:58:26.569363 22444 ts_manager.cc:194] Registered new tserver with Master: 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:26.570070 22444 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.130:38109
W20240502 07:58:26.660598 22766 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:26.660827 22766 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:26.660867 22766 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:26.663715 22766 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:26.663765 22766 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:26.663822 22766 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.131
I20240502 07:58:26.667064 22766 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:41971
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.16.85.131
--webserver_port=0
--tserver_master_addrs=127.16.85.190:39083
--never_fsync=true
--heap_profile_path=/tmp/kudu.22766
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.131
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:26.667737 22766 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:26.668648 22766 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:26.676223 22774 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:26.676810 22772 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:26.677373 22771 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:26.678231 22766 server_base.cc:1034] running on GCE node
I20240502 07:58:26.678591 22766 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:26.679049 22766 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:26.680150 22766 hybrid_clock.cc:648] HybridClock initialized: now 1714636706680139 us; error 27 us; skew 500 ppm
I20240502 07:58:26.685899 22766 webserver.cc:460] Webserver started at http://127.16.85.131:37873/ using document root <none> and password file <none>
I20240502 07:58:26.686398 22766 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:26.686450 22766 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:26.686609 22766 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:26.688060 22766 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "7db4178eed174225bfb24f5f9445ce88"
format_stamp: "Formatted at 2024-05-02 07:58:26 on dist-test-slave-7df7"
I20240502 07:58:26.688482 22766 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "7db4178eed174225bfb24f5f9445ce88"
format_stamp: "Formatted at 2024-05-02 07:58:26 on dist-test-slave-7df7"
I20240502 07:58:26.691473 22766 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:26.693331 22780 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:26.694149 22766 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:26.694232 22766 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "7db4178eed174225bfb24f5f9445ce88"
format_stamp: "Formatted at 2024-05-02 07:58:26 on dist-test-slave-7df7"
I20240502 07:58:26.694306 22766 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:26.702807 22766 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:26.703236 22766 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:26.703357 22766 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:26.703752 22766 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:26.704532 22766 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:26.704581 22766 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:26.704630 22766 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:26.704691 22766 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:26.712530 22766 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.131:44055
I20240502 07:58:26.712563 22893 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.131:44055 every 8 connection(s)
I20240502 07:58:26.713388 22766 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 07:58:26.720665 22894 heartbeater.cc:347] Connected to a master server at 127.16.85.190:39083
I20240502 07:58:26.720844 22894 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:26.721308 22894 heartbeater.cc:511] Master 127.16.85.190:39083 requested a full tablet report, sending...
I20240502 07:58:26.721798 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 22766
I20240502 07:58:26.721901 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 07:58:26.722168 22444 ts_manager.cc:194] Registered new tserver with Master: 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:26.722703 22444 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.131:33007
I20240502 07:58:26.723780 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.132:0
--local_ip_for_outbound_sockets=127.16.85.132
--webserver_interface=127.16.85.132
--webserver_port=0
--tserver_master_addrs=127.16.85.190:39083
--builtin_ntp_servers=127.16.85.148:41971
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 07:58:26.819343 22897 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:26.819518 22897 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:26.819556 22897 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:26.822408 22897 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:26.822456 22897 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:26.822513 22897 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.132
I20240502 07:58:26.825620 22897 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:41971
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.132:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.16.85.132
--webserver_port=0
--tserver_master_addrs=127.16.85.190:39083
--never_fsync=true
--heap_profile_path=/tmp/kudu.22897
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.132
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:26.826357 22897 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:26.827358 22897 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:26.835201 22903 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:26.835894 22902 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:26.836700 22905 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:26.837320 22897 server_base.cc:1034] running on GCE node
I20240502 07:58:26.837641 22897 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:26.838121 22897 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:26.839255 22897 hybrid_clock.cc:648] HybridClock initialized: now 1714636706839243 us; error 24 us; skew 500 ppm
I20240502 07:58:26.840960 22897 webserver.cc:460] Webserver started at http://127.16.85.132:33133/ using document root <none> and password file <none>
I20240502 07:58:26.841455 22897 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:26.841504 22897 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:26.841645 22897 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:26.843143 22897 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "02a92e1c084b4308b9e855a4cc2c295f"
format_stamp: "Formatted at 2024-05-02 07:58:26 on dist-test-slave-7df7"
I20240502 07:58:26.843518 22897 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "02a92e1c084b4308b9e855a4cc2c295f"
format_stamp: "Formatted at 2024-05-02 07:58:26 on dist-test-slave-7df7"
I20240502 07:58:26.846379 22897 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.003s
I20240502 07:58:26.848191 22911 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:26.849085 22897 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:26.849174 22897 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "02a92e1c084b4308b9e855a4cc2c295f"
format_stamp: "Formatted at 2024-05-02 07:58:26 on dist-test-slave-7df7"
I20240502 07:58:26.849237 22897 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:26.871255 22897 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:26.871702 22897 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:26.871820 22897 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:26.872244 22897 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:26.873005 22897 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:26.873070 22897 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:26.873117 22897 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:26.873145 22897 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:26.880122 22897 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.132:39425
I20240502 07:58:26.880165 23024 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.132:39425 every 8 connection(s)
I20240502 07:58:26.880901 22897 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 07:58:26.887701 23025 heartbeater.cc:347] Connected to a master server at 127.16.85.190:39083
I20240502 07:58:26.887862 23025 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:26.888255 23025 heartbeater.cc:511] Master 127.16.85.190:39083 requested a full tablet report, sending...
I20240502 07:58:26.888674 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 22897
I20240502 07:58:26.888787 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 07:58:26.889128 22440 ts_manager.cc:194] Registered new tserver with Master: 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:26.889685 22440 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.132:43373
I20240502 07:58:26.890655 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.133:0
--local_ip_for_outbound_sockets=127.16.85.133
--webserver_interface=127.16.85.133
--webserver_port=0
--tserver_master_addrs=127.16.85.190:39083
--builtin_ntp_servers=127.16.85.148:41971
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 07:58:26.985167 23028 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:26.985322 23028 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:26.985359 23028 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:26.987874 23028 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:26.987917 23028 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:26.987987 23028 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.133
I20240502 07:58:26.991214 23028 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:41971
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.133:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.16.85.133
--webserver_port=0
--tserver_master_addrs=127.16.85.190:39083
--never_fsync=true
--heap_profile_path=/tmp/kudu.23028
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.133
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:26.991879 23028 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:26.992811 23028 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:26.999953 23033 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:27.000805 23036 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:27.001466 23034 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:27.002071 23028 server_base.cc:1034] running on GCE node
I20240502 07:58:27.002410 23028 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:27.002849 23028 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:27.003983 23028 hybrid_clock.cc:648] HybridClock initialized: now 1714636707003971 us; error 38 us; skew 500 ppm
I20240502 07:58:27.005668 23028 webserver.cc:460] Webserver started at http://127.16.85.133:36449/ using document root <none> and password file <none>
I20240502 07:58:27.006140 23028 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:27.006191 23028 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:27.006366 23028 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:27.007807 23028 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "6d02017318a2461796d431962fe6a537"
format_stamp: "Formatted at 2024-05-02 07:58:27 on dist-test-slave-7df7"
I20240502 07:58:27.008200 23028 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "6d02017318a2461796d431962fe6a537"
format_stamp: "Formatted at 2024-05-02 07:58:27 on dist-test-slave-7df7"
I20240502 07:58:27.011049 23028 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 07:58:27.013098 23042 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:27.013916 23028 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:27.014006 23028 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "6d02017318a2461796d431962fe6a537"
format_stamp: "Formatted at 2024-05-02 07:58:27 on dist-test-slave-7df7"
I20240502 07:58:27.014070 23028 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:27.028021 23028 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:27.028496 23028 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:27.028628 23028 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:27.029182 23028 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:27.029965 23028 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:27.030009 23028 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:27.030063 23028 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:27.030088 23028 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:27.037475 23028 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.133:35459
I20240502 07:58:27.037518 23155 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.133:35459 every 8 connection(s)
I20240502 07:58:27.038239 23028 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 07:58:27.045405 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 23028
I20240502 07:58:27.045430 23156 heartbeater.cc:347] Connected to a master server at 127.16.85.190:39083
I20240502 07:58:27.045521 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 07:58:27.045610 23156 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:27.046108 23156 heartbeater.cc:511] Master 127.16.85.190:39083 requested a full tablet report, sending...
I20240502 07:58:27.046880 22444 ts_manager.cc:194] Registered new tserver with Master: 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:27.047508 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.134:0
--local_ip_for_outbound_sockets=127.16.85.134
--webserver_interface=127.16.85.134
--webserver_port=0
--tserver_master_addrs=127.16.85.190:39083
--builtin_ntp_servers=127.16.85.148:41971
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:27.047513 22444 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.133:44129
W20240502 07:58:27.144093 23159 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:27.144264 23159 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:27.144302 23159 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:27.146996 23159 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:27.147048 23159 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:27.147117 23159 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.134
I20240502 07:58:27.150167 23159 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:41971
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.134:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.16.85.134
--webserver_port=0
--tserver_master_addrs=127.16.85.190:39083
--never_fsync=true
--heap_profile_path=/tmp/kudu.23159
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.134
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:27.150859 23159 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:27.151798 23159 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:27.159149 23167 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:27.159722 23165 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:27.160287 23164 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:27.161003 23159 server_base.cc:1034] running on GCE node
I20240502 07:58:27.161347 23159 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:27.161784 23159 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:27.162909 23159 hybrid_clock.cc:648] HybridClock initialized: now 1714636707162897 us; error 34 us; skew 500 ppm
I20240502 07:58:27.164521 23159 webserver.cc:460] Webserver started at http://127.16.85.134:46069/ using document root <none> and password file <none>
I20240502 07:58:27.164996 23159 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:27.165045 23159 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:27.165202 23159 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:27.166579 23159 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/instance:
uuid: "74f9aa91c2c448f980b512fe2e8d672e"
format_stamp: "Formatted at 2024-05-02 07:58:27 on dist-test-slave-7df7"
I20240502 07:58:27.167001 23159 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/instance:
uuid: "74f9aa91c2c448f980b512fe2e8d672e"
format_stamp: "Formatted at 2024-05-02 07:58:27 on dist-test-slave-7df7"
I20240502 07:58:27.169930 23159 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.001s sys 0.004s
I20240502 07:58:27.171764 23173 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:27.172607 23159 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:27.172721 23159 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "74f9aa91c2c448f980b512fe2e8d672e"
format_stamp: "Formatted at 2024-05-02 07:58:27 on dist-test-slave-7df7"
I20240502 07:58:27.172787 23159 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:27.180876 23159 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:27.181349 23159 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:27.181480 23159 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:27.181898 23159 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:27.182637 23159 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:27.182678 23159 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:27.182724 23159 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:27.182751 23159 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:27.189697 23159 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.134:34857
I20240502 07:58:27.189718 23286 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.134:34857 every 8 connection(s)
I20240502 07:58:27.190455 23159 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 07:58:27.192021 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 23159
I20240502 07:58:27.192135 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240502 07:58:27.194218 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.135:0
--local_ip_for_outbound_sockets=127.16.85.135
--webserver_interface=127.16.85.135
--webserver_port=0
--tserver_master_addrs=127.16.85.190:39083
--builtin_ntp_servers=127.16.85.148:41971
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:27.199347 23287 heartbeater.cc:347] Connected to a master server at 127.16.85.190:39083
I20240502 07:58:27.199512 23287 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:27.199945 23287 heartbeater.cc:511] Master 127.16.85.190:39083 requested a full tablet report, sending...
I20240502 07:58:27.200773 22444 ts_manager.cc:194] Registered new tserver with Master: 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:27.201426 22444 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.134:44027
W20240502 07:58:27.291568 23290 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:27.291751 23290 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:27.291815 23290 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:27.294515 23290 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:27.294574 23290 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:27.294656 23290 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.135
I20240502 07:58:27.297782 23290 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:41971
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.135:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.16.85.135
--webserver_port=0
--tserver_master_addrs=127.16.85.190:39083
--never_fsync=true
--heap_profile_path=/tmp/kudu.23290
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.135
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:27.298489 23290 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:27.299434 23290 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:27.306943 23295 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:27.307767 23298 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:27.308363 23296 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:27.308995 23290 server_base.cc:1034] running on GCE node
I20240502 07:58:27.309289 23290 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:27.309739 23290 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:27.310873 23290 hybrid_clock.cc:648] HybridClock initialized: now 1714636707310865 us; error 21 us; skew 500 ppm
I20240502 07:58:27.312681 23290 webserver.cc:460] Webserver started at http://127.16.85.135:36171/ using document root <none> and password file <none>
I20240502 07:58:27.313167 23290 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:27.313228 23290 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:27.313398 23290 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:27.314745 23290 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/instance:
uuid: "4654231dfe994c1a9d2e37d9c3d75699"
format_stamp: "Formatted at 2024-05-02 07:58:27 on dist-test-slave-7df7"
I20240502 07:58:27.315151 23290 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/instance:
uuid: "4654231dfe994c1a9d2e37d9c3d75699"
format_stamp: "Formatted at 2024-05-02 07:58:27 on dist-test-slave-7df7"
I20240502 07:58:27.317932 23290 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 07:58:27.319715 23304 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:27.320540 23290 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:27.320627 23290 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "4654231dfe994c1a9d2e37d9c3d75699"
format_stamp: "Formatted at 2024-05-02 07:58:27 on dist-test-slave-7df7"
I20240502 07:58:27.320720 23290 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:27.348282 23290 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:27.348773 23290 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:27.348906 23290 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:27.349376 23290 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:27.350131 23290 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:27.350172 23290 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:27.350227 23290 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:27.350251 23290 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:27.357712 23290 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.135:41205
I20240502 07:58:27.357731 23417 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.135:41205 every 8 connection(s)
I20240502 07:58:27.358503 23290 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 07:58:27.359481 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 23290
I20240502 07:58:27.359591 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/instance
I20240502 07:58:27.361642 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.136:0
--local_ip_for_outbound_sockets=127.16.85.136
--webserver_interface=127.16.85.136
--webserver_port=0
--tserver_master_addrs=127.16.85.190:39083
--builtin_ntp_servers=127.16.85.148:41971
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:27.367159 23418 heartbeater.cc:347] Connected to a master server at 127.16.85.190:39083
I20240502 07:58:27.367326 23418 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:27.367751 23418 heartbeater.cc:511] Master 127.16.85.190:39083 requested a full tablet report, sending...
I20240502 07:58:27.368598 22444 ts_manager.cc:194] Registered new tserver with Master: 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:27.369323 22444 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.135:32825
I20240502 07:58:27.416759 22632 heartbeater.cc:503] Master 127.16.85.190:39083 was elected leader, sending a full tablet report...
W20240502 07:58:27.454789 23421 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:27.454941 23421 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:27.454977 23421 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:27.457456 23421 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:27.457499 23421 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:27.457552 23421 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.136
I20240502 07:58:27.460546 23421 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:41971
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.136:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.16.85.136
--webserver_port=0
--tserver_master_addrs=127.16.85.190:39083
--never_fsync=true
--heap_profile_path=/tmp/kudu.23421
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.136
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:27.461239 23421 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:27.462168 23421 file_cache.cc:492] Constructed file cache file cache with capacity 419430
I20240502 07:58:27.469498 23421 server_base.cc:1034] running on GCE node
W20240502 07:58:27.470067 23426 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:27.470734 23429 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:27.471428 23427 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:27.471781 23421 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:27.472247 23421 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:27.473392 23421 hybrid_clock.cc:648] HybridClock initialized: now 1714636707473383 us; error 22 us; skew 500 ppm
I20240502 07:58:27.474949 23421 webserver.cc:460] Webserver started at http://127.16.85.136:36521/ using document root <none> and password file <none>
I20240502 07:58:27.475416 23421 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:27.475466 23421 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:27.475615 23421 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 07:58:27.477068 23421 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data/instance:
uuid: "7eced220cf1642f3873fc385b6191e69"
format_stamp: "Formatted at 2024-05-02 07:58:27 on dist-test-slave-7df7"
I20240502 07:58:27.477478 23421 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/instance:
uuid: "7eced220cf1642f3873fc385b6191e69"
format_stamp: "Formatted at 2024-05-02 07:58:27 on dist-test-slave-7df7"
I20240502 07:58:27.480182 23421 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 07:58:27.482069 23435 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:27.482901 23421 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:27.482985 23421 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "7eced220cf1642f3873fc385b6191e69"
format_stamp: "Formatted at 2024-05-02 07:58:27 on dist-test-slave-7df7"
I20240502 07:58:27.483059 23421 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:27.503356 23421 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:27.503772 23421 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:27.503913 23421 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:27.504360 23421 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:27.505059 23421 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:27.505111 23421 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:27.505149 23421 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:27.505167 23421 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:27.512473 23421 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.136:34769
I20240502 07:58:27.512506 23548 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.136:34769 every 8 connection(s)
I20240502 07:58:27.513273 23421 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240502 07:58:27.516463 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 23421
I20240502 07:58:27.516561 16726 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/instance
I20240502 07:58:27.520326 23549 heartbeater.cc:347] Connected to a master server at 127.16.85.190:39083
I20240502 07:58:27.520484 23549 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:27.520913 23549 heartbeater.cc:511] Master 127.16.85.190:39083 requested a full tablet report, sending...
I20240502 07:58:27.521728 22444 ts_manager.cc:194] Registered new tserver with Master: 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:27.522193 22444 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.136:60283
I20240502 07:58:27.529891 16726 external_mini_cluster.cc:930] 8 TS(s) registered with all masters
I20240502 07:58:27.537550 16726 external_mini_cluster.cc:930] 8 TS(s) registered with all masters
I20240502 07:58:27.537606 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 23028
I20240502 07:58:27.542798 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 23159
I20240502 07:58:27.547806 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 23290
I20240502 07:58:27.553547 16726 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0LLEln/build/debug/bin/kudu with pid 23421
I20240502 07:58:27.571734 22763 heartbeater.cc:503] Master 127.16.85.190:39083 was elected leader, sending a full tablet report...
I20240502 07:58:27.724478 22894 heartbeater.cc:503] Master 127.16.85.190:39083 was elected leader, sending a full tablet report...
I20240502 07:58:27.891641 23025 heartbeater.cc:503] Master 127.16.85.190:39083 was elected leader, sending a full tablet report...
I20240502 07:58:31.309468 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:31.328047 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 6844411c7a99471f8f5bbe36ed89f91e (DEFAULT_TABLE table=rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.328045 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 6844411c7a99471f8f5bbe36ed89f91e (DEFAULT_TABLE table=rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.328045 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 6844411c7a99471f8f5bbe36ed89f91e (DEFAULT_TABLE table=rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.328334 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 2dd6748f1e8a4308b89e0329c54d63ac (DEFAULT_TABLE table=rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.328964 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6844411c7a99471f8f5bbe36ed89f91e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.328964 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2dd6748f1e8a4308b89e0329c54d63ac. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.328964 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6844411c7a99471f8f5bbe36ed89f91e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.328970 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6844411c7a99471f8f5bbe36ed89f91e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.329694 22696 tablet_service.cc:1467] Processing CreateTablet for tablet 2dd6748f1e8a4308b89e0329c54d63ac (DEFAULT_TABLE table=rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.329929 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2dd6748f1e8a4308b89e0329c54d63ac. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.330121 22958 tablet_service.cc:1467] Processing CreateTablet for tablet 207d85c84a6e4929b4842990860c65e6 (DEFAULT_TABLE table=rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.330354 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 207d85c84a6e4929b4842990860c65e6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.330756 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 207d85c84a6e4929b4842990860c65e6 (DEFAULT_TABLE table=rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.330976 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 207d85c84a6e4929b4842990860c65e6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.331393 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 207d85c84a6e4929b4842990860c65e6 (DEFAULT_TABLE table=rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.331651 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 207d85c84a6e4929b4842990860c65e6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.332851 22565 tablet_service.cc:1467] Processing CreateTablet for tablet 2dd6748f1e8a4308b89e0329c54d63ac (DEFAULT_TABLE table=rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.333055 22565 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2dd6748f1e8a4308b89e0329c54d63ac. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.338660 23569 tablet_bootstrap.cc:492] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:31.339113 23571 tablet_bootstrap.cc:492] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:31.339286 23572 tablet_bootstrap.cc:492] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:31.340809 23572 tablet_bootstrap.cc:654] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.340809 23569 tablet_bootstrap.cc:654] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.340811 23571 tablet_bootstrap.cc:654] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.341413 23571 log.cc:824] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:31.341529 23569 log.cc:824] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:31.341529 23572 log.cc:824] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:31.342064 23570 tablet_bootstrap.cc:492] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:31.343062 23569 tablet_bootstrap.cc:492] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:31.343062 23572 tablet_bootstrap.cc:492] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:31.343065 23571 tablet_bootstrap.cc:492] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:31.343216 23571 ts_tablet_manager.cc:1367] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
I20240502 07:58:31.343221 23569 ts_tablet_manager.cc:1367] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.004s
I20240502 07:58:31.343216 23572 ts_tablet_manager.cc:1367] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
I20240502 07:58:31.343876 23570 tablet_bootstrap.cc:654] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.344543 23570 log.cc:824] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:31.346266 23570 tablet_bootstrap.cc:492] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:31.346419 23570 ts_tablet_manager.cc:1367] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 07:58:31.346784 23569 raft_consensus.cc:348] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.346760 23572 raft_consensus.cc:348] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.347187 23572 raft_consensus.cc:374] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.347255 23572 raft_consensus.cc:729] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.347303 23569 raft_consensus.cc:374] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.347406 23569 raft_consensus.cc:729] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.347584 23571 raft_consensus.cc:348] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.347775 23572 consensus_queue.cc:260] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.347775 23569 consensus_queue.cc:260] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.347872 23571 raft_consensus.cc:374] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.347927 23571 raft_consensus.cc:729] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.348315 23571 consensus_queue.cc:260] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.348560 23569 ts_tablet_manager.cc:1398] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.005s user 0.000s sys 0.004s
I20240502 07:58:31.348560 23572 ts_tablet_manager.cc:1398] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.005s user 0.004s sys 0.002s
I20240502 07:58:31.348901 23569 tablet_bootstrap.cc:492] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:31.348901 23572 tablet_bootstrap.cc:492] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:31.349753 23569 tablet_bootstrap.cc:654] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.349753 23572 tablet_bootstrap.cc:654] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.349923 23570 raft_consensus.cc:348] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.350102 23570 raft_consensus.cc:374] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.350160 23570 raft_consensus.cc:729] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.350540 23570 consensus_queue.cc:260] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.351286 23570 ts_tablet_manager.cc:1398] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.005s user 0.005s sys 0.000s
I20240502 07:58:31.351958 23572 tablet_bootstrap.cc:492] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:31.352079 23572 ts_tablet_manager.cc:1367] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:31.352469 23570 tablet_bootstrap.cc:492] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:31.352634 23569 tablet_bootstrap.cc:492] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:31.352767 23569 ts_tablet_manager.cc:1367] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:31.352783 23572 raft_consensus.cc:348] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.353082 23572 raft_consensus.cc:374] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.353118 23572 raft_consensus.cc:729] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.353245 23572 consensus_queue.cc:260] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.353272 23569 raft_consensus.cc:348] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.353367 23569 raft_consensus.cc:374] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.353395 23569 raft_consensus.cc:729] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.353463 23570 tablet_bootstrap.cc:654] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.353490 23569 consensus_queue.cc:260] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.353571 23572 ts_tablet_manager.cc:1398] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:31.353719 23569 ts_tablet_manager.cc:1398] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:31.354827 23571 ts_tablet_manager.cc:1398] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.012s user 0.004s sys 0.002s
I20240502 07:58:31.354892 23570 tablet_bootstrap.cc:492] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:31.355012 23570 ts_tablet_manager.cc:1367] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:31.355100 23571 tablet_bootstrap.cc:492] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:31.355523 23570 raft_consensus.cc:348] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.355643 23570 raft_consensus.cc:374] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.355700 23570 raft_consensus.cc:729] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.355825 23570 consensus_queue.cc:260] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.356146 23571 tablet_bootstrap.cc:654] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.356213 23570 ts_tablet_manager.cc:1398] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:31.356384 23570 tablet_bootstrap.cc:492] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:31.357194 23570 tablet_bootstrap.cc:654] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.358381 23571 tablet_bootstrap.cc:492] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:31.358472 23571 ts_tablet_manager.cc:1367] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:31.358882 23571 raft_consensus.cc:348] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.358968 23571 raft_consensus.cc:374] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.359000 23571 raft_consensus.cc:729] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.359122 23571 consensus_queue.cc:260] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.359376 23571 ts_tablet_manager.cc:1398] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:31.360056 23570 tablet_bootstrap.cc:492] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:31.360133 23570 ts_tablet_manager.cc:1367] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:31.360503 23570 raft_consensus.cc:348] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.360587 23570 raft_consensus.cc:374] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.360618 23570 raft_consensus.cc:729] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.360745 23570 consensus_queue.cc:260] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.360962 23570 ts_tablet_manager.cc:1398] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
W20240502 07:58:31.384950 23026 tablet.cc:2369] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 07:58:31.409365 22633 tablet.cc:2369] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 07:58:31.459545 23579 raft_consensus.cc:482] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:31.459740 23579 raft_consensus.cc:504] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.460954 23579 leader_election.cc:288] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:31.464509 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "207d85c84a6e4929b4842990860c65e6" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:31.464531 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "207d85c84a6e4929b4842990860c65e6" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:31.464763 22586 raft_consensus.cc:2454] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:31.464766 22848 raft_consensus.cc:2454] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:31.465083 22915 leader_election.cc:302] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:31.465260 23579 raft_consensus.cc:2781] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:31.465323 23579 raft_consensus.cc:482] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:31.465365 23579 raft_consensus.cc:3037] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:31.466099 23579 raft_consensus.cc:504] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.466351 23579 leader_election.cc:288] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:31.466583 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "207d85c84a6e4929b4842990860c65e6" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:31.466667 22848 raft_consensus.cc:3037] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:31.466672 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "207d85c84a6e4929b4842990860c65e6" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:31.466770 22586 raft_consensus.cc:3037] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
W20240502 07:58:31.467291 22895 tablet.cc:2369] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 07:58:31.467446 22848 raft_consensus.cc:2454] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:31.467597 22586 raft_consensus.cc:2454] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:31.467660 22915 leader_election.cc:302] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:31.467814 23579 raft_consensus.cc:2781] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:31.467959 23579 raft_consensus.cc:686] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:31.468191 23579 consensus_queue.cc:237] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.470575 22442 catalog_manager.cc:5484] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:31.482565 23582 raft_consensus.cc:482] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:31.482678 23582 raft_consensus.cc:504] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.483556 23582 leader_election.cc:288] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:31.487336 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2dd6748f1e8a4308b89e0329c54d63ac" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:31.487449 22848 raft_consensus.cc:2454] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:31.487454 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2dd6748f1e8a4308b89e0329c54d63ac" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:31.487711 22717 raft_consensus.cc:2454] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:31.487730 22522 leader_election.cc:302] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:31.487926 23582 raft_consensus.cc:2781] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:31.487977 23582 raft_consensus.cc:482] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:31.488000 23582 raft_consensus.cc:3037] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:31.488788 23582 raft_consensus.cc:504] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.489065 23582 leader_election.cc:288] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:31.489329 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2dd6748f1e8a4308b89e0329c54d63ac" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:31.489331 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2dd6748f1e8a4308b89e0329c54d63ac" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:31.489423 22848 raft_consensus.cc:3037] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:31.489416 22717 raft_consensus.cc:3037] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:31.490109 22848 raft_consensus.cc:2454] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:31.490201 22717 raft_consensus.cc:2454] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:31.490339 22522 leader_election.cc:302] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:31.490486 23582 raft_consensus.cc:2781] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:31.490695 23582 raft_consensus.cc:686] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:31.490978 23582 consensus_queue.cc:237] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.493726 22442 catalog_manager.cc:5484] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:31.536157 23579 raft_consensus.cc:482] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:31.536303 23579 raft_consensus.cc:504] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.536818 23579 leader_election.cc:288] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:31.537324 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6844411c7a99471f8f5bbe36ed89f91e" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:31.537465 22586 raft_consensus.cc:2454] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:31.537712 22915 leader_election.cc:302] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:31.537846 23579 raft_consensus.cc:2781] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:31.537891 23579 raft_consensus.cc:482] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:31.537933 23579 raft_consensus.cc:3037] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:31.538632 23579 raft_consensus.cc:504] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.538882 23579 leader_election.cc:288] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:31.539180 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6844411c7a99471f8f5bbe36ed89f91e" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:31.539274 22586 raft_consensus.cc:3037] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:31.540100 22586 raft_consensus.cc:2454] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:31.540302 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6844411c7a99471f8f5bbe36ed89f91e" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:31.540391 22915 leader_election.cc:302] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:31.540371 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6844411c7a99471f8f5bbe36ed89f91e" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:31.540419 22717 raft_consensus.cc:2454] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:31.540542 23579 raft_consensus.cc:2781] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:31.540624 23579 raft_consensus.cc:686] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:31.540758 23579 consensus_queue.cc:237] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:31.542197 22442 catalog_manager.cc:5484] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:31.564170 22764 tablet.cc:2369] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 07:58:31.655848 23595 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:31.656085 23595 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:31.658728 23595 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_0'
I20240502 07:58:31.749459 22585 raft_consensus.cc:1261] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:31.749552 22848 raft_consensus.cc:1261] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:31.749460 22586 raft_consensus.cc:1261] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:31.750157 23579 consensus_queue.cc:1035] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:31.750384 23627 consensus_queue.cc:1035] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:31.750658 22716 raft_consensus.cc:3037] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:31.750754 23579 consensus_queue.cc:1035] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:31.751797 22716 raft_consensus.cc:1261] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:31.752372 23588 consensus_queue.cc:1035] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:31.757292 22717 raft_consensus.cc:1261] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:31.757517 22848 raft_consensus.cc:1261] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:31.757807 23625 mvcc.cc:205] Tried to move back new op lower bound from 7023151971320045568 to 7023151970175766528. Current Snapshot: MvccSnapshot[applied={T|T < 7023151971320045568}]
I20240502 07:58:31.757886 23582 consensus_queue.cc:1035] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:31.759898 23592 consensus_queue.cc:1035] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:31.762908 23629 mvcc.cc:205] Tried to move back new op lower bound from 7023151971320045568 to 7023151970175766528. Current Snapshot: MvccSnapshot[applied={T|T < 7023151971320045568}]
I20240502 07:58:31.775822 23641 mvcc.cc:205] Tried to move back new op lower bound from 7023151971352682496 to 7023151970269335552. Current Snapshot: MvccSnapshot[applied={T|T < 7023151971352682496}]
I20240502 07:58:31.778486 23626 mvcc.cc:205] Tried to move back new op lower bound from 7023151971318337536 to 7023151970472153088. Current Snapshot: MvccSnapshot[applied={T|T < 7023151971318337536}]
INSERT report
rows total: 2000
time total: 109.944 ms
time per row: 0.0549719 ms
I20240502 07:58:31.819061 22441 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:31.824138 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 313eef41dee34cf2863004373a449156 (DEFAULT_TABLE table=rebalance_test_table_1 [id=ed9ff98b498847f8a18550017ae8fc8c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.824189 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 313eef41dee34cf2863004373a449156 (DEFAULT_TABLE table=rebalance_test_table_1 [id=ed9ff98b498847f8a18550017ae8fc8c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.824189 22827 tablet_service.cc:1467] Processing CreateTablet for tablet e5d5f67e026344998d5fb079f532d076 (DEFAULT_TABLE table=rebalance_test_table_1 [id=ed9ff98b498847f8a18550017ae8fc8c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.824450 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 313eef41dee34cf2863004373a449156. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.824527 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e5d5f67e026344998d5fb079f532d076. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.825186 22959 tablet_service.cc:1467] Processing CreateTablet for tablet e5d5f67e026344998d5fb079f532d076 (DEFAULT_TABLE table=rebalance_test_table_1 [id=ed9ff98b498847f8a18550017ae8fc8c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.825348 22826 tablet_service.cc:1467] Processing CreateTablet for tablet 015ed33d8c634473997eaa60e91269a2 (DEFAULT_TABLE table=rebalance_test_table_1 [id=ed9ff98b498847f8a18550017ae8fc8c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.825426 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e5d5f67e026344998d5fb079f532d076. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.825579 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 015ed33d8c634473997eaa60e91269a2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.826746 22958 tablet_service.cc:1467] Processing CreateTablet for tablet 313eef41dee34cf2863004373a449156 (DEFAULT_TABLE table=rebalance_test_table_1 [id=ed9ff98b498847f8a18550017ae8fc8c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.826925 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 313eef41dee34cf2863004373a449156. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.827252 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 313eef41dee34cf2863004373a449156. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.827476 23571 tablet_bootstrap.cc:492] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:31.828289 23571 tablet_bootstrap.cc:654] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.828923 23569 tablet_bootstrap.cc:492] T 313eef41dee34cf2863004373a449156 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:31.829200 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 015ed33d8c634473997eaa60e91269a2 (DEFAULT_TABLE table=rebalance_test_table_1 [id=ed9ff98b498847f8a18550017ae8fc8c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.829408 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 015ed33d8c634473997eaa60e91269a2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.829603 23569 tablet_bootstrap.cc:654] T 313eef41dee34cf2863004373a449156 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.830184 23572 tablet_bootstrap.cc:492] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:31.830629 23569 tablet_bootstrap.cc:492] T 313eef41dee34cf2863004373a449156 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:31.830701 23569 ts_tablet_manager.cc:1367] T 313eef41dee34cf2863004373a449156 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:31.830875 23572 tablet_bootstrap.cc:654] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.831223 23569 raft_consensus.cc:348] T 313eef41dee34cf2863004373a449156 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:31.831319 23570 tablet_bootstrap.cc:492] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:31.831357 23569 raft_consensus.cc:374] T 313eef41dee34cf2863004373a449156 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.831396 23569 raft_consensus.cc:729] T 313eef41dee34cf2863004373a449156 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.831511 23569 consensus_queue.cc:260] T 313eef41dee34cf2863004373a449156 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:31.831810 23569 ts_tablet_manager.cc:1398] T 313eef41dee34cf2863004373a449156 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:31.832124 23570 tablet_bootstrap.cc:654] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.832363 23572 tablet_bootstrap.cc:492] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:31.832368 22696 tablet_service.cc:1467] Processing CreateTablet for tablet 015ed33d8c634473997eaa60e91269a2 (DEFAULT_TABLE table=rebalance_test_table_1 [id=ed9ff98b498847f8a18550017ae8fc8c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.832433 23572 ts_tablet_manager.cc:1367] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:31.832602 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 015ed33d8c634473997eaa60e91269a2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.832870 23572 raft_consensus.cc:348] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.832944 23572 raft_consensus.cc:374] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.832963 23572 raft_consensus.cc:729] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.833060 23572 consensus_queue.cc:260] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.833282 23572 ts_tablet_manager.cc:1398] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:31.833418 23572 tablet_bootstrap.cc:492] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:31.833794 23570 tablet_bootstrap.cc:492] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:31.833859 23570 ts_tablet_manager.cc:1367] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.001s
I20240502 07:58:31.834142 23572 tablet_bootstrap.cc:654] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.834228 23570 raft_consensus.cc:348] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:31.834314 23570 raft_consensus.cc:374] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.834343 23570 raft_consensus.cc:729] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.834442 23570 consensus_queue.cc:260] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:31.834679 23570 ts_tablet_manager.cc:1398] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:31.835090 23572 tablet_bootstrap.cc:492] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:31.835168 23572 ts_tablet_manager.cc:1367] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:31.835546 23572 raft_consensus.cc:348] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:31.835657 23572 raft_consensus.cc:374] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.835686 23572 raft_consensus.cc:729] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.835789 23572 consensus_queue.cc:260] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:31.836046 23571 tablet_bootstrap.cc:492] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:31.836062 23572 ts_tablet_manager.cc:1398] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:31.836126 23571 ts_tablet_manager.cc:1367] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.009s user 0.000s sys 0.001s
I20240502 07:58:31.836202 23572 tablet_bootstrap.cc:492] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:31.835533 22697 tablet_service.cc:1467] Processing CreateTablet for tablet e5d5f67e026344998d5fb079f532d076 (DEFAULT_TABLE table=rebalance_test_table_1 [id=ed9ff98b498847f8a18550017ae8fc8c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:31.836547 23571 raft_consensus.cc:348] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.836676 23571 raft_consensus.cc:374] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.836747 23571 raft_consensus.cc:729] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.836831 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e5d5f67e026344998d5fb079f532d076. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:31.836869 23571 consensus_queue.cc:260] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.837092 23571 ts_tablet_manager.cc:1398] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:31.837244 23571 tablet_bootstrap.cc:492] T 313eef41dee34cf2863004373a449156 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:31.837374 23572 tablet_bootstrap.cc:654] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.837915 23571 tablet_bootstrap.cc:654] T 313eef41dee34cf2863004373a449156 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.838584 23572 tablet_bootstrap.cc:492] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:31.838707 23572 ts_tablet_manager.cc:1367] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240502 07:58:31.839184 23572 raft_consensus.cc:348] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:31.834532 23569 tablet_bootstrap.cc:492] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:31.839330 23572 raft_consensus.cc:374] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.839380 23572 raft_consensus.cc:729] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.839449 23571 tablet_bootstrap.cc:492] T 313eef41dee34cf2863004373a449156 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:31.839515 23571 ts_tablet_manager.cc:1367] T 313eef41dee34cf2863004373a449156 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:31.839514 23572 consensus_queue.cc:260] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:31.839766 23572 ts_tablet_manager.cc:1398] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:31.839912 23571 raft_consensus.cc:348] T 313eef41dee34cf2863004373a449156 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:31.840010 23571 raft_consensus.cc:374] T 313eef41dee34cf2863004373a449156 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.840039 23571 raft_consensus.cc:729] T 313eef41dee34cf2863004373a449156 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.840140 23571 consensus_queue.cc:260] T 313eef41dee34cf2863004373a449156 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:31.840361 23571 ts_tablet_manager.cc:1398] T 313eef41dee34cf2863004373a449156 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:31.840554 23569 tablet_bootstrap.cc:654] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.841656 23569 tablet_bootstrap.cc:492] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:31.841743 23569 ts_tablet_manager.cc:1367] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 07:58:31.842425 23569 raft_consensus.cc:348] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:31.842531 23569 raft_consensus.cc:374] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.842578 23569 raft_consensus.cc:729] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.842715 23569 consensus_queue.cc:260] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:31.842969 23569 ts_tablet_manager.cc:1398] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:31.843115 23569 tablet_bootstrap.cc:492] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:31.843817 23569 tablet_bootstrap.cc:654] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:31.845084 23569 tablet_bootstrap.cc:492] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:31.845160 23569 ts_tablet_manager.cc:1367] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:31.845505 23569 raft_consensus.cc:348] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.845590 23569 raft_consensus.cc:374] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:31.845615 23569 raft_consensus.cc:729] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:31.845726 23569 consensus_queue.cc:260] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.845971 23569 ts_tablet_manager.cc:1398] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:31.896911 23578 raft_consensus.cc:482] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:31.897063 23578 raft_consensus.cc:504] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:31.898084 23578 leader_election.cc:288] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:31.902169 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "313eef41dee34cf2863004373a449156" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:31.902207 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "313eef41dee34cf2863004373a449156" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:31.902308 22716 raft_consensus.cc:2454] T 313eef41dee34cf2863004373a449156 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:31.902379 22979 raft_consensus.cc:2454] T 313eef41dee34cf2863004373a449156 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:31.902647 22781 leader_election.cc:302] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:31.902848 23578 raft_consensus.cc:2781] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:31.902902 23578 raft_consensus.cc:482] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:31.902958 23578 raft_consensus.cc:3037] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:31.903721 23578 raft_consensus.cc:504] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:31.904021 23578 leader_election.cc:288] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:31.904274 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "313eef41dee34cf2863004373a449156" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:31.904381 22716 raft_consensus.cc:3037] T 313eef41dee34cf2863004373a449156 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:31.904408 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "313eef41dee34cf2863004373a449156" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:31.904511 22979 raft_consensus.cc:3037] T 313eef41dee34cf2863004373a449156 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:31.905172 22716 raft_consensus.cc:2454] T 313eef41dee34cf2863004373a449156 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:31.905292 22979 raft_consensus.cc:2454] T 313eef41dee34cf2863004373a449156 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:31.905417 22781 leader_election.cc:302] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:31.905568 23578 raft_consensus.cc:2781] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:31.905753 23578 raft_consensus.cc:686] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:31.906008 23578 consensus_queue.cc:237] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:31.907900 22444 catalog_manager.cc:5484] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:31.926925 23627 raft_consensus.cc:482] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:31.927034 23627 raft_consensus.cc:504] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.927407 23627 leader_election.cc:288] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:31.927762 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e5d5f67e026344998d5fb079f532d076" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:31.927788 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e5d5f67e026344998d5fb079f532d076" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:31.927886 22716 raft_consensus.cc:2454] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:31.927886 22848 raft_consensus.cc:2454] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:31.928150 22915 leader_election.cc:302] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:31.928274 23627 raft_consensus.cc:2781] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:31.928328 23627 raft_consensus.cc:482] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:31.928352 23627 raft_consensus.cc:3037] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:31.929139 23627 raft_consensus.cc:504] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.929386 23627 leader_election.cc:288] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:31.929646 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e5d5f67e026344998d5fb079f532d076" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:31.929652 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e5d5f67e026344998d5fb079f532d076" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:31.929733 22848 raft_consensus.cc:3037] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:31.929751 22716 raft_consensus.cc:3037] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:31.930527 22716 raft_consensus.cc:2454] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:31.930552 22848 raft_consensus.cc:2454] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:31.930794 22912 leader_election.cc:302] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:31.930946 23578 raft_consensus.cc:482] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:31.930971 23627 raft_consensus.cc:2781] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:31.931063 23627 raft_consensus.cc:686] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:31.931064 23578 raft_consensus.cc:504] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:31.931221 23627 consensus_queue.cc:237] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:31.931552 23578 leader_election.cc:288] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:31.931852 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "015ed33d8c634473997eaa60e91269a2" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:31.931957 22716 raft_consensus.cc:2454] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:31.932369 22781 leader_election.cc:302] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:31.932621 23578 raft_consensus.cc:2781] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:31.932710 23578 raft_consensus.cc:482] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:31.932781 23578 raft_consensus.cc:3037] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:31.933459 22444 catalog_manager.cc:5484] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:31.933607 23578 raft_consensus.cc:504] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:31.934571 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "015ed33d8c634473997eaa60e91269a2" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:31.934662 22716 raft_consensus.cc:3037] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:31.935180 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "015ed33d8c634473997eaa60e91269a2" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:31.935191 22585 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "015ed33d8c634473997eaa60e91269a2" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:31.935335 22586 raft_consensus.cc:2454] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:31.935643 22716 raft_consensus.cc:2454] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:31.935864 22781 leader_election.cc:302] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters: 8bea0fe8d8044a2685f92f7a10c9cf85
I20240502 07:58:31.934020 23578 leader_election.cc:288] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:31.936038 23670 raft_consensus.cc:2781] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:31.936225 23670 raft_consensus.cc:686] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:31.936345 23670 consensus_queue.cc:237] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:31.937706 22444 catalog_manager.cc:5484] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } }
W20240502 07:58:32.047753 23675 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:32.047955 23675 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:32.050745 23675 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_1'
I20240502 07:58:32.116397 22716 raft_consensus.cc:1261] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.116843 23628 consensus_queue.cc:1035] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:32.117722 22848 raft_consensus.cc:1261] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.118098 22717 raft_consensus.cc:1261] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.118726 23670 consensus_queue.cc:1035] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:32.119830 22586 raft_consensus.cc:3037] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.119836 22979 raft_consensus.cc:1261] T 313eef41dee34cf2863004373a449156 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.119827 23628 consensus_queue.cc:1035] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:32.120204 22716 raft_consensus.cc:1261] T 313eef41dee34cf2863004373a449156 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.120270 23670 consensus_queue.cc:1035] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:32.120843 22586 raft_consensus.cc:1261] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.125198 23578 consensus_queue.cc:1035] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:32.125507 23578 consensus_queue.cc:1035] T 313eef41dee34cf2863004373a449156 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 89.8192 ms
time per row: 0.0449096 ms
I20240502 07:58:32.177557 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:32.182314 22564 tablet_service.cc:1467] Processing CreateTablet for tablet d7a82b58251f498cba0f0134785bfe5b (DEFAULT_TABLE table=rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.182416 22958 tablet_service.cc:1467] Processing CreateTablet for tablet d7a82b58251f498cba0f0134785bfe5b (DEFAULT_TABLE table=rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.182619 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d7a82b58251f498cba0f0134785bfe5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.182694 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d7a82b58251f498cba0f0134785bfe5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.183367 22828 tablet_service.cc:1467] Processing CreateTablet for tablet be8ac3cda68f4f3a8e07f913094f3b0e (DEFAULT_TABLE table=rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.183624 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet be8ac3cda68f4f3a8e07f913094f3b0e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.184239 23570 tablet_bootstrap.cc:492] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:32.184592 23571 tablet_bootstrap.cc:492] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:32.184840 22697 tablet_service.cc:1467] Processing CreateTablet for tablet d7a82b58251f498cba0f0134785bfe5b (DEFAULT_TABLE table=rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.185077 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d7a82b58251f498cba0f0134785bfe5b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.185117 23570 tablet_bootstrap.cc:654] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.185431 23571 tablet_bootstrap.cc:654] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.185910 23572 tablet_bootstrap.cc:492] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:32.186507 22959 tablet_service.cc:1467] Processing CreateTablet for tablet b9bd4ccf02cf471685c02a402d883bb5 (DEFAULT_TABLE table=rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.186676 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b9bd4ccf02cf471685c02a402d883bb5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.186691 23572 tablet_bootstrap.cc:654] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.186992 23569 tablet_bootstrap.cc:492] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:32.187172 22696 tablet_service.cc:1467] Processing CreateTablet for tablet b9bd4ccf02cf471685c02a402d883bb5 (DEFAULT_TABLE table=rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.187383 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b9bd4ccf02cf471685c02a402d883bb5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.187777 23569 tablet_bootstrap.cc:654] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.187819 23572 tablet_bootstrap.cc:492] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:32.187882 23572 ts_tablet_manager.cc:1367] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.001s
I20240502 07:58:32.188313 23572 raft_consensus.cc:348] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.188438 23572 raft_consensus.cc:374] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.188474 23572 raft_consensus.cc:729] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.188581 23572 consensus_queue.cc:260] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.188820 23572 ts_tablet_manager.cc:1398] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:32.189153 23571 tablet_bootstrap.cc:492] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:32.189249 23571 ts_tablet_manager.cc:1367] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:32.189693 23571 raft_consensus.cc:348] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.189787 23569 tablet_bootstrap.cc:492] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:32.189796 23571 raft_consensus.cc:374] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.189827 23571 raft_consensus.cc:729] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.189857 23569 ts_tablet_manager.cc:1367] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.001s
I20240502 07:58:32.189930 23571 consensus_queue.cc:260] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.190044 22565 tablet_service.cc:1467] Processing CreateTablet for tablet b9bd4ccf02cf471685c02a402d883bb5 (DEFAULT_TABLE table=rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.190155 23571 ts_tablet_manager.cc:1398] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:32.190274 22565 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b9bd4ccf02cf471685c02a402d883bb5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.190237 23569 raft_consensus.cc:348] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.190320 23571 tablet_bootstrap.cc:492] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:32.190320 23569 raft_consensus.cc:374] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.190352 23569 raft_consensus.cc:729] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.190472 23569 consensus_queue.cc:260] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.190745 23569 ts_tablet_manager.cc:1398] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:32.191308 23571 tablet_bootstrap.cc:654] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.192013 22958 tablet_service.cc:1467] Processing CreateTablet for tablet be8ac3cda68f4f3a8e07f913094f3b0e (DEFAULT_TABLE table=rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.192253 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet be8ac3cda68f4f3a8e07f913094f3b0e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.193734 23571 tablet_bootstrap.cc:492] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:32.193819 23571 ts_tablet_manager.cc:1367] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:32.194205 23571 raft_consensus.cc:348] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.194310 23571 raft_consensus.cc:374] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.194355 23571 raft_consensus.cc:729] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.194469 23571 consensus_queue.cc:260] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.194715 23571 ts_tablet_manager.cc:1398] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:32.194876 23571 tablet_bootstrap.cc:492] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:32.195006 23569 tablet_bootstrap.cc:492] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:32.195696 23571 tablet_bootstrap.cc:654] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.195780 23569 tablet_bootstrap.cc:654] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.196892 22564 tablet_service.cc:1467] Processing CreateTablet for tablet be8ac3cda68f4f3a8e07f913094f3b0e (DEFAULT_TABLE table=rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.197083 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet be8ac3cda68f4f3a8e07f913094f3b0e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.199015 23571 tablet_bootstrap.cc:492] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:32.199076 23571 ts_tablet_manager.cc:1367] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:32.199148 23570 tablet_bootstrap.cc:492] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:32.199215 23570 ts_tablet_manager.cc:1367] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.015s user 0.000s sys 0.002s
I20240502 07:58:32.199321 23571 raft_consensus.cc:348] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.199393 23571 raft_consensus.cc:374] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.199432 23571 raft_consensus.cc:729] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.199498 23571 consensus_queue.cc:260] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.199647 23571 ts_tablet_manager.cc:1398] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:32.199604 23570 raft_consensus.cc:348] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.199687 23570 raft_consensus.cc:374] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.199712 23570 raft_consensus.cc:729] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.199841 23570 consensus_queue.cc:260] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.200054 23570 ts_tablet_manager.cc:1398] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:32.200176 23570 tablet_bootstrap.cc:492] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:32.201228 23570 tablet_bootstrap.cc:654] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.203114 23569 tablet_bootstrap.cc:492] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:32.203183 23569 ts_tablet_manager.cc:1367] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.008s user 0.000s sys 0.001s
I20240502 07:58:32.203511 23569 raft_consensus.cc:348] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.203598 23569 raft_consensus.cc:374] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.203619 23569 raft_consensus.cc:729] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.203714 23569 consensus_queue.cc:260] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.203910 23569 ts_tablet_manager.cc:1398] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:32.204639 23570 tablet_bootstrap.cc:492] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:32.204722 23570 ts_tablet_manager.cc:1367] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:32.205041 23570 raft_consensus.cc:348] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.205130 23570 raft_consensus.cc:374] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.205154 23570 raft_consensus.cc:729] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.205269 23570 consensus_queue.cc:260] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.205497 23570 ts_tablet_manager.cc:1398] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:32.205610 23570 tablet_bootstrap.cc:492] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:32.206298 23570 tablet_bootstrap.cc:654] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.207501 23570 tablet_bootstrap.cc:492] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:32.207572 23570 ts_tablet_manager.cc:1367] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:32.207895 23570 raft_consensus.cc:348] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.207991 23570 raft_consensus.cc:374] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.208027 23570 raft_consensus.cc:729] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.208130 23570 consensus_queue.cc:260] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.208379 23570 ts_tablet_manager.cc:1398] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:32.250694 23577 raft_consensus.cc:482] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:32.250823 23577 raft_consensus.cc:504] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.251858 23577 leader_election.cc:288] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:32.255183 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d7a82b58251f498cba0f0134785bfe5b" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:32.255229 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d7a82b58251f498cba0f0134785bfe5b" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:32.255344 22979 raft_consensus.cc:2454] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:32.255369 22586 raft_consensus.cc:2454] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:32.255651 22651 leader_election.cc:302] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:32.255859 23577 raft_consensus.cc:2781] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:32.255911 23577 raft_consensus.cc:482] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:32.255936 23577 raft_consensus.cc:3037] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.256702 23577 raft_consensus.cc:504] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.256978 23577 leader_election.cc:288] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:32.257262 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d7a82b58251f498cba0f0134785bfe5b" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:32.257263 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d7a82b58251f498cba0f0134785bfe5b" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:32.257361 22979 raft_consensus.cc:3037] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.257361 22586 raft_consensus.cc:3037] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.258255 22979 raft_consensus.cc:2454] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:32.258255 22586 raft_consensus.cc:2454] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:32.258493 22651 leader_election.cc:302] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:32.258692 23577 raft_consensus.cc:2781] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:32.258864 23577 raft_consensus.cc:686] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:32.259150 23577 consensus_queue.cc:237] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.260998 22442 catalog_manager.cc:5484] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:32.268784 23673 raft_consensus.cc:482] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:32.268885 23673 raft_consensus.cc:504] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.269232 23673 leader_election.cc:288] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:32.269541 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "be8ac3cda68f4f3a8e07f913094f3b0e" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:32.269541 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "be8ac3cda68f4f3a8e07f913094f3b0e" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:32.269657 22979 raft_consensus.cc:2454] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:32.269657 22586 raft_consensus.cc:2454] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:32.269901 22784 leader_election.cc:302] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:32.270054 23673 raft_consensus.cc:2781] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:32.270108 23673 raft_consensus.cc:482] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:32.270135 23673 raft_consensus.cc:3037] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.270876 23673 raft_consensus.cc:504] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.271170 23673 leader_election.cc:288] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:32.271446 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "be8ac3cda68f4f3a8e07f913094f3b0e" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:32.271456 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "be8ac3cda68f4f3a8e07f913094f3b0e" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:32.271528 22979 raft_consensus.cc:3037] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.271539 22586 raft_consensus.cc:3037] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.272186 22979 raft_consensus.cc:2454] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:32.272224 22586 raft_consensus.cc:2454] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:32.272722 22782 leader_election.cc:302] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:32.272878 23673 raft_consensus.cc:2781] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:32.272958 23673 raft_consensus.cc:686] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:32.273079 23673 consensus_queue.cc:237] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.274401 22442 catalog_manager.cc:5484] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:32.350389 23577 raft_consensus.cc:482] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:32.350522 23577 raft_consensus.cc:504] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.350900 23577 leader_election.cc:288] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:32.351265 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b9bd4ccf02cf471685c02a402d883bb5" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:32.351311 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b9bd4ccf02cf471685c02a402d883bb5" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:32.351394 22586 raft_consensus.cc:2454] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:32.351452 22979 raft_consensus.cc:2454] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:32.351748 22651 leader_election.cc:302] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:32.351920 23577 raft_consensus.cc:2781] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:32.351976 23577 raft_consensus.cc:482] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:32.352020 23577 raft_consensus.cc:3037] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.352916 23577 raft_consensus.cc:504] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.353237 23577 leader_election.cc:288] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:32.353507 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b9bd4ccf02cf471685c02a402d883bb5" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:32.353519 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b9bd4ccf02cf471685c02a402d883bb5" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:32.353610 22979 raft_consensus.cc:3037] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.353621 22586 raft_consensus.cc:3037] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.354276 22979 raft_consensus.cc:2454] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:32.354279 22586 raft_consensus.cc:2454] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:32.354488 22651 leader_election.cc:302] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:32.354665 23577 raft_consensus.cc:2781] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:32.354729 23577 raft_consensus.cc:686] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:32.354849 23577 consensus_queue.cc:237] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.356211 22442 catalog_manager.cc:5484] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:32.478289 23725 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:32.478513 23725 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:32.481475 23725 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_2'
I20240502 07:58:32.549798 22979 raft_consensus.cc:1261] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.550343 23670 consensus_queue.cc:1035] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:32.550424 22586 raft_consensus.cc:1261] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.550832 23670 consensus_queue.cc:1035] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:32.556397 22979 raft_consensus.cc:1261] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.556896 22979 raft_consensus.cc:1261] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.556875 23721 consensus_queue.cc:1035] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:32.557034 22585 raft_consensus.cc:1261] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.557381 23721 consensus_queue.cc:1035] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:32.557492 23577 consensus_queue.cc:1035] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:32.582109 22585 raft_consensus.cc:1261] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.582705 23577 consensus_queue.cc:1035] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 86.3326 ms
time per row: 0.0431663 ms
I20240502 07:58:32.605158 22441 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:32.611500 22826 tablet_service.cc:1467] Processing CreateTablet for tablet ca85b40df34d4e1c91a55d7dd69e1494 (DEFAULT_TABLE table=rebalance_test_table_3 [id=8f253ed078dd4e948cfdd6ff6c19f874]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.611822 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ca85b40df34d4e1c91a55d7dd69e1494. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.611799 22827 tablet_service.cc:1467] Processing CreateTablet for tablet e24e4e639b0649e3ac135ac9358a0398 (DEFAULT_TABLE table=rebalance_test_table_3 [id=8f253ed078dd4e948cfdd6ff6c19f874]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.611982 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e24e4e639b0649e3ac135ac9358a0398. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.612901 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 68f96ca3354645fd91f666109eeaf847 (DEFAULT_TABLE table=rebalance_test_table_3 [id=8f253ed078dd4e948cfdd6ff6c19f874]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.612941 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 68f96ca3354645fd91f666109eeaf847 (DEFAULT_TABLE table=rebalance_test_table_3 [id=8f253ed078dd4e948cfdd6ff6c19f874]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.613101 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 68f96ca3354645fd91f666109eeaf847. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.613163 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 68f96ca3354645fd91f666109eeaf847. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.614058 22696 tablet_service.cc:1467] Processing CreateTablet for tablet 68f96ca3354645fd91f666109eeaf847 (DEFAULT_TABLE table=rebalance_test_table_3 [id=8f253ed078dd4e948cfdd6ff6c19f874]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.614324 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 68f96ca3354645fd91f666109eeaf847. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.615180 22565 tablet_service.cc:1467] Processing CreateTablet for tablet ca85b40df34d4e1c91a55d7dd69e1494 (DEFAULT_TABLE table=rebalance_test_table_3 [id=8f253ed078dd4e948cfdd6ff6c19f874]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.615379 22565 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ca85b40df34d4e1c91a55d7dd69e1494. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.615473 22958 tablet_service.cc:1467] Processing CreateTablet for tablet ca85b40df34d4e1c91a55d7dd69e1494 (DEFAULT_TABLE table=rebalance_test_table_3 [id=8f253ed078dd4e948cfdd6ff6c19f874]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.615669 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ca85b40df34d4e1c91a55d7dd69e1494. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.616396 22697 tablet_service.cc:1467] Processing CreateTablet for tablet e24e4e639b0649e3ac135ac9358a0398 (DEFAULT_TABLE table=rebalance_test_table_3 [id=8f253ed078dd4e948cfdd6ff6c19f874]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.616578 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e24e4e639b0649e3ac135ac9358a0398. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.616993 23572 tablet_bootstrap.cc:492] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:32.617434 22959 tablet_service.cc:1467] Processing CreateTablet for tablet e24e4e639b0649e3ac135ac9358a0398 (DEFAULT_TABLE table=rebalance_test_table_3 [id=8f253ed078dd4e948cfdd6ff6c19f874]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:32.617609 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e24e4e639b0649e3ac135ac9358a0398. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:32.617831 23572 tablet_bootstrap.cc:654] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.618355 23569 tablet_bootstrap.cc:492] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:32.618362 23570 tablet_bootstrap.cc:492] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:32.619069 23570 tablet_bootstrap.cc:654] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.619069 23569 tablet_bootstrap.cc:654] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.619093 23571 tablet_bootstrap.cc:492] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:32.619812 23571 tablet_bootstrap.cc:654] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.620874 23571 tablet_bootstrap.cc:492] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:32.620950 23571 ts_tablet_manager.cc:1367] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:32.621384 23571 raft_consensus.cc:348] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:32.621495 23571 raft_consensus.cc:374] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.621533 23571 raft_consensus.cc:729] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.621639 23571 consensus_queue.cc:260] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:32.621721 23569 tablet_bootstrap.cc:492] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:32.621790 23569 ts_tablet_manager.cc:1367] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.001s
I20240502 07:58:32.621941 23571 ts_tablet_manager.cc:1398] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:32.622068 23571 tablet_bootstrap.cc:492] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:32.622193 23569 raft_consensus.cc:348] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.622275 23569 raft_consensus.cc:374] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.622306 23569 raft_consensus.cc:729] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.622423 23569 consensus_queue.cc:260] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.622706 23569 ts_tablet_manager.cc:1398] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:32.622855 23569 tablet_bootstrap.cc:492] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:32.623005 23571 tablet_bootstrap.cc:654] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.623749 23569 tablet_bootstrap.cc:654] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.624003 23571 tablet_bootstrap.cc:492] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:32.624066 23571 ts_tablet_manager.cc:1367] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:32.624466 23571 raft_consensus.cc:348] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.624550 23571 raft_consensus.cc:374] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.624586 23571 raft_consensus.cc:729] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.624699 23571 consensus_queue.cc:260] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.624919 23571 ts_tablet_manager.cc:1398] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:32.625790 23570 tablet_bootstrap.cc:492] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:32.625885 23570 ts_tablet_manager.cc:1367] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.000s
I20240502 07:58:32.626271 23570 raft_consensus.cc:348] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.626374 23570 raft_consensus.cc:374] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.626425 23570 raft_consensus.cc:729] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.626535 23570 consensus_queue.cc:260] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.626616 23569 tablet_bootstrap.cc:492] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:32.626677 23569 ts_tablet_manager.cc:1367] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.001s
I20240502 07:58:32.626783 23570 ts_tablet_manager.cc:1398] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:32.626924 23570 tablet_bootstrap.cc:492] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:32.627010 23569 raft_consensus.cc:348] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.627110 23569 raft_consensus.cc:374] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.627135 23569 raft_consensus.cc:729] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.627228 23569 consensus_queue.cc:260] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.627445 23569 ts_tablet_manager.cc:1398] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:32.627667 23570 tablet_bootstrap.cc:654] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.628808 23570 tablet_bootstrap.cc:492] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:32.628885 23570 ts_tablet_manager.cc:1367] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:32.629249 23570 raft_consensus.cc:348] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:32.629323 23570 raft_consensus.cc:374] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.629349 23570 raft_consensus.cc:729] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.629442 23570 consensus_queue.cc:260] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:32.629604 23572 tablet_bootstrap.cc:492] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:32.629671 23570 ts_tablet_manager.cc:1398] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:32.629673 23572 ts_tablet_manager.cc:1367] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.013s user 0.001s sys 0.000s
I20240502 07:58:32.630064 23572 raft_consensus.cc:348] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:32.630158 23572 raft_consensus.cc:374] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.630192 23572 raft_consensus.cc:729] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.630299 23572 consensus_queue.cc:260] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:32.630586 23572 ts_tablet_manager.cc:1398] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:32.630726 23572 tablet_bootstrap.cc:492] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:32.631512 23572 tablet_bootstrap.cc:654] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.634450 23572 tablet_bootstrap.cc:492] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:32.634524 23572 ts_tablet_manager.cc:1367] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:32.634857 23572 raft_consensus.cc:348] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.634936 23572 raft_consensus.cc:374] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.634966 23572 raft_consensus.cc:729] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.635188 23572 consensus_queue.cc:260] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.635463 23572 ts_tablet_manager.cc:1398] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:32.635594 23572 tablet_bootstrap.cc:492] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:32.636760 23572 tablet_bootstrap.cc:654] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:32.637607 23572 tablet_bootstrap.cc:492] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:32.637679 23572 ts_tablet_manager.cc:1367] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:32.637969 23572 raft_consensus.cc:348] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.638041 23572 raft_consensus.cc:374] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:32.638072 23572 raft_consensus.cc:729] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:32.638180 23572 consensus_queue.cc:260] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.638404 23572 ts_tablet_manager.cc:1398] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:32.677181 23628 raft_consensus.cc:482] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:32.677294 23628 raft_consensus.cc:504] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.677611 23628 leader_election.cc:288] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:32.677930 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e24e4e639b0649e3ac135ac9358a0398" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:32.677945 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e24e4e639b0649e3ac135ac9358a0398" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:32.678037 22848 raft_consensus.cc:2454] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:32.678067 22717 raft_consensus.cc:2454] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:32.678288 22915 leader_election.cc:302] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:32.678442 23628 raft_consensus.cc:2781] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:32.678493 23628 raft_consensus.cc:482] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:32.678537 23628 raft_consensus.cc:3037] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.679252 23628 raft_consensus.cc:504] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.679545 23628 leader_election.cc:288] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:32.679798 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e24e4e639b0649e3ac135ac9358a0398" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:32.679847 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e24e4e639b0649e3ac135ac9358a0398" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:32.679890 22848 raft_consensus.cc:3037] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.679987 22717 raft_consensus.cc:3037] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.680603 22848 raft_consensus.cc:2454] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:32.680785 22717 raft_consensus.cc:2454] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:32.680840 22915 leader_election.cc:302] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:32.680984 23628 raft_consensus.cc:2781] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:32.681113 23628 raft_consensus.cc:686] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:32.681228 23628 consensus_queue.cc:237] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:32.682634 22442 catalog_manager.cc:5484] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:32.740581 23756 raft_consensus.cc:482] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:32.740737 23756 raft_consensus.cc:504] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.741127 23756 leader_election.cc:288] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:32.741804 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "68f96ca3354645fd91f666109eeaf847" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:32.741945 22586 raft_consensus.cc:2454] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:32.742277 22653 leader_election.cc:302] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:32.742437 23756 raft_consensus.cc:2781] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:32.742482 23756 raft_consensus.cc:482] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:32.742530 23756 raft_consensus.cc:3037] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.743294 23756 raft_consensus.cc:504] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.743638 23756 leader_election.cc:288] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:32.744029 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "68f96ca3354645fd91f666109eeaf847" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:32.744153 22586 raft_consensus.cc:3037] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.744424 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "68f96ca3354645fd91f666109eeaf847" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:32.744462 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "68f96ca3354645fd91f666109eeaf847" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:32.744563 22848 raft_consensus.cc:2454] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:32.745040 22586 raft_consensus.cc:2454] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:32.745260 22653 leader_election.cc:302] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters: 7db4178eed174225bfb24f5f9445ce88
I20240502 07:58:32.745427 23756 raft_consensus.cc:2781] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:32.745522 23756 raft_consensus.cc:686] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:32.745673 23756 consensus_queue.cc:237] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:32.747187 22442 catalog_manager.cc:5484] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:32.761588 23628 raft_consensus.cc:482] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:32.761677 23628 raft_consensus.cc:504] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:32.762032 23628 leader_election.cc:288] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:32.762383 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca85b40df34d4e1c91a55d7dd69e1494" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:32.762418 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca85b40df34d4e1c91a55d7dd69e1494" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:32.762527 22586 raft_consensus.cc:2454] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:32.762532 22847 raft_consensus.cc:2454] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:32.762771 22915 leader_election.cc:302] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:32.762950 23628 raft_consensus.cc:2781] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:32.762996 23628 raft_consensus.cc:482] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:32.763018 23628 raft_consensus.cc:3037] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.763710 23628 raft_consensus.cc:504] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:32.763955 23628 leader_election.cc:288] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:32.764251 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca85b40df34d4e1c91a55d7dd69e1494" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:32.764317 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca85b40df34d4e1c91a55d7dd69e1494" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:32.764345 22847 raft_consensus.cc:3037] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.764417 22586 raft_consensus.cc:3037] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.765102 22586 raft_consensus.cc:2454] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:32.765111 22847 raft_consensus.cc:2454] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:32.765348 22915 leader_election.cc:302] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:32.765491 23628 raft_consensus.cc:2781] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:32.765558 23628 raft_consensus.cc:686] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:32.765674 23628 consensus_queue.cc:237] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:32.767053 22442 catalog_manager.cc:5484] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:32.877992 23775 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:32.878202 23775 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:32.881122 23775 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_3'
I20240502 07:58:32.952625 22847 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.952875 22717 raft_consensus.cc:1261] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.952924 22586 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.953210 23771 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:32.953306 23628 consensus_queue.cc:1035] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:32.953676 22847 raft_consensus.cc:1261] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.957731 23628 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:32.957820 23771 consensus_queue.cc:1035] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:32.967569 22584 raft_consensus.cc:1261] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.967960 23756 consensus_queue.cc:1035] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:32.968744 22847 raft_consensus.cc:3037] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:32.969743 22847 raft_consensus.cc:1261] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:32.975981 23756 consensus_queue.cc:1035] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 97.4356 ms
time per row: 0.0487178 ms
I20240502 07:58:33.019879 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_4"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:33.024940 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 9cf91afea4ed460bb478a031f9520590 (DEFAULT_TABLE table=rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.025022 22565 tablet_service.cc:1467] Processing CreateTablet for tablet 9cf91afea4ed460bb478a031f9520590 (DEFAULT_TABLE table=rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.025099 22696 tablet_service.cc:1467] Processing CreateTablet for tablet 9cf91afea4ed460bb478a031f9520590 (DEFAULT_TABLE table=rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.025238 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9cf91afea4ed460bb478a031f9520590. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.025308 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9cf91afea4ed460bb478a031f9520590. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.025305 22565 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9cf91afea4ed460bb478a031f9520590. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.025852 22959 tablet_service.cc:1467] Processing CreateTablet for tablet c5853c8107214c618e2ba9d6d983de2f (DEFAULT_TABLE table=rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.026087 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c5853c8107214c618e2ba9d6d983de2f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.027607 23572 tablet_bootstrap.cc:492] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:33.028081 23571 tablet_bootstrap.cc:492] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:33.028189 22958 tablet_service.cc:1467] Processing CreateTablet for tablet 76c97fb98dd949b9b65fa02d77968cfa (DEFAULT_TABLE table=rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.028400 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 76c97fb98dd949b9b65fa02d77968cfa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.028445 23572 tablet_bootstrap.cc:654] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.028924 23571 tablet_bootstrap.cc:654] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.029630 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 76c97fb98dd949b9b65fa02d77968cfa (DEFAULT_TABLE table=rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.029781 23572 tablet_bootstrap.cc:492] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:33.029842 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 76c97fb98dd949b9b65fa02d77968cfa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.029843 22564 tablet_service.cc:1467] Processing CreateTablet for tablet c5853c8107214c618e2ba9d6d983de2f (DEFAULT_TABLE table=rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.030020 23569 tablet_bootstrap.cc:492] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:33.030054 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c5853c8107214c618e2ba9d6d983de2f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.030862 23570 tablet_bootstrap.cc:492] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:33.030964 23569 tablet_bootstrap.cc:654] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.031725 23570 tablet_bootstrap.cc:654] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.029852 23572 ts_tablet_manager.cc:1367] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:33.032063 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 76c97fb98dd949b9b65fa02d77968cfa (DEFAULT_TABLE table=rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.032177 23572 raft_consensus.cc:348] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.032290 23572 raft_consensus.cc:374] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.032297 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 76c97fb98dd949b9b65fa02d77968cfa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.032326 23572 raft_consensus.cc:729] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.032447 23572 consensus_queue.cc:260] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.032729 23569 tablet_bootstrap.cc:492] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:33.032759 23572 ts_tablet_manager.cc:1398] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:33.032801 23569 ts_tablet_manager.cc:1367] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.001s
I20240502 07:58:33.032938 23572 tablet_bootstrap.cc:492] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:33.033185 22696 tablet_service.cc:1467] Processing CreateTablet for tablet c5853c8107214c618e2ba9d6d983de2f (DEFAULT_TABLE table=rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.033214 23569 raft_consensus.cc:348] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.033334 23569 raft_consensus.cc:374] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.033365 23569 raft_consensus.cc:729] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.033393 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c5853c8107214c618e2ba9d6d983de2f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.033491 23569 consensus_queue.cc:260] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.033771 23572 tablet_bootstrap.cc:654] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.033792 23569 ts_tablet_manager.cc:1398] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:33.034847 23571 tablet_bootstrap.cc:492] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:33.034922 23571 ts_tablet_manager.cc:1367] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
I20240502 07:58:33.035326 23571 raft_consensus.cc:348] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.035398 23570 tablet_bootstrap.cc:492] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:33.035414 23571 raft_consensus.cc:374] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.035447 23571 raft_consensus.cc:729] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.035465 23570 ts_tablet_manager.cc:1367] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:33.035552 23571 consensus_queue.cc:260] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.035817 23571 ts_tablet_manager.cc:1398] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:33.035874 23569 tablet_bootstrap.cc:492] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:33.035873 23570 raft_consensus.cc:348] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.035960 23571 tablet_bootstrap.cc:492] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:33.035962 23570 raft_consensus.cc:374] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.035992 23570 raft_consensus.cc:729] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.036100 23570 consensus_queue.cc:260] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.036358 23570 ts_tablet_manager.cc:1398] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:33.036497 23570 tablet_bootstrap.cc:492] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:33.036710 23569 tablet_bootstrap.cc:654] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.037086 23571 tablet_bootstrap.cc:654] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.037367 23570 tablet_bootstrap.cc:654] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.038491 23572 tablet_bootstrap.cc:492] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:33.038551 23571 tablet_bootstrap.cc:492] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:33.038564 23572 ts_tablet_manager.cc:1367] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:58:33.038626 23571 ts_tablet_manager.cc:1367] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:33.038934 23572 raft_consensus.cc:348] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.039019 23572 raft_consensus.cc:374] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.039007 23571 raft_consensus.cc:348] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.039060 23572 raft_consensus.cc:729] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.039088 23571 raft_consensus.cc:374] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.039119 23571 raft_consensus.cc:729] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.039175 23572 consensus_queue.cc:260] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.039224 23571 consensus_queue.cc:260] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.039425 23572 ts_tablet_manager.cc:1398] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:33.039462 23571 ts_tablet_manager.cc:1398] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:33.040709 23570 tablet_bootstrap.cc:492] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:33.040781 23570 ts_tablet_manager.cc:1367] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:33.041189 23570 raft_consensus.cc:348] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.041281 23570 raft_consensus.cc:374] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.041318 23570 raft_consensus.cc:729] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.041422 23570 consensus_queue.cc:260] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.041611 23569 tablet_bootstrap.cc:492] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:33.041782 23570 ts_tablet_manager.cc:1398] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:33.041960 23570 tablet_bootstrap.cc:492] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:33.042819 23570 tablet_bootstrap.cc:654] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.043383 23569 ts_tablet_manager.cc:1367] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.001s
I20240502 07:58:33.043841 23569 raft_consensus.cc:348] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.043921 23569 raft_consensus.cc:374] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.043954 23569 raft_consensus.cc:729] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.044092 23569 consensus_queue.cc:260] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.044126 23570 tablet_bootstrap.cc:492] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:33.044198 23570 ts_tablet_manager.cc:1367] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:33.044345 23569 ts_tablet_manager.cc:1398] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:33.044605 23570 raft_consensus.cc:348] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.044765 23570 raft_consensus.cc:374] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.044802 23570 raft_consensus.cc:729] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.044934 23570 consensus_queue.cc:260] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.045215 23570 ts_tablet_manager.cc:1398] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:33.132400 23654 raft_consensus.cc:482] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:33.132546 23654 raft_consensus.cc:504] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.133052 23654 leader_election.cc:288] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:33.133364 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76c97fb98dd949b9b65fa02d77968cfa" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:33.133517 22847 raft_consensus.cc:2454] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:33.133777 22522 leader_election.cc:302] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:33.133952 23654 raft_consensus.cc:2781] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:33.133996 23654 raft_consensus.cc:482] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:33.134019 23654 raft_consensus.cc:3037] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.135041 23654 raft_consensus.cc:504] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.135389 23654 leader_election.cc:288] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:33.135663 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76c97fb98dd949b9b65fa02d77968cfa" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:33.135763 22847 raft_consensus.cc:3037] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.136633 22847 raft_consensus.cc:2454] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:33.136732 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76c97fb98dd949b9b65fa02d77968cfa" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:33.136739 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76c97fb98dd949b9b65fa02d77968cfa" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:33.136891 22978 raft_consensus.cc:2454] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:33.136965 22522 leader_election.cc:302] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:33.137128 23654 raft_consensus.cc:2781] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:33.137280 23654 raft_consensus.cc:686] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:33.137423 23654 consensus_queue.cc:237] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.138976 22435 catalog_manager.cc:5484] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:33.278869 23673 raft_consensus.cc:482] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:33.279003 23673 raft_consensus.cc:504] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.279394 23673 leader_election.cc:288] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:33.279816 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cf91afea4ed460bb478a031f9520590" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:33.279961 22586 raft_consensus.cc:2454] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:33.280203 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cf91afea4ed460bb478a031f9520590" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:33.280331 22717 raft_consensus.cc:2454] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:33.280359 22784 leader_election.cc:302] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:33.280534 23673 raft_consensus.cc:2781] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:33.280594 23673 raft_consensus.cc:482] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:33.280620 23673 raft_consensus.cc:3037] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.281387 23673 raft_consensus.cc:504] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.281663 23673 leader_election.cc:288] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:33.281980 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cf91afea4ed460bb478a031f9520590" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:33.282001 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cf91afea4ed460bb478a031f9520590" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:33.282063 22717 raft_consensus.cc:3037] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.282106 22586 raft_consensus.cc:3037] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.282869 22717 raft_consensus.cc:2454] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:33.282879 22586 raft_consensus.cc:2454] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:33.283129 22784 leader_election.cc:302] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:33.283313 23673 raft_consensus.cc:2781] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:33.283393 23673 raft_consensus.cc:686] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:33.283533 23673 consensus_queue.cc:237] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.285048 22435 catalog_manager.cc:5484] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:33.295399 23654 raft_consensus.cc:482] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:33.295501 23654 raft_consensus.cc:504] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.295857 23654 leader_election.cc:288] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:33.296083 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c5853c8107214c618e2ba9d6d983de2f" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:33.296197 22717 raft_consensus.cc:2454] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:33.296226 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c5853c8107214c618e2ba9d6d983de2f" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:33.296370 22979 raft_consensus.cc:2454] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:33.296447 22519 leader_election.cc:302] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:33.296599 23654 raft_consensus.cc:2781] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:33.296680 23654 raft_consensus.cc:482] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:33.296730 23654 raft_consensus.cc:3037] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.297454 23654 raft_consensus.cc:504] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.297714 23654 leader_election.cc:288] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:33.297976 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c5853c8107214c618e2ba9d6d983de2f" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:33.298079 22979 raft_consensus.cc:3037] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.298055 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c5853c8107214c618e2ba9d6d983de2f" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:33.298151 22717 raft_consensus.cc:3037] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.298781 22717 raft_consensus.cc:2454] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:33.298796 22979 raft_consensus.cc:2454] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:33.298993 22519 leader_election.cc:302] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:33.299185 23654 raft_consensus.cc:2781] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:33.299257 23654 raft_consensus.cc:686] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:33.299391 23654 consensus_queue.cc:237] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:33.300781 22435 catalog_manager.cc:5484] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:33.423381 23830 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:33.423628 23830 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:33.426690 23830 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_4'
I20240502 07:58:33.503858 22717 raft_consensus.cc:1261] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:33.504225 23654 consensus_queue.cc:1035] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:33.504894 22979 raft_consensus.cc:1261] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:33.505333 23654 consensus_queue.cc:1035] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:33.509132 22717 raft_consensus.cc:1261] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:33.509259 22586 raft_consensus.cc:1261] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:33.509559 23673 consensus_queue.cc:1035] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:33.509668 23670 consensus_queue.cc:1035] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:33.516093 22977 raft_consensus.cc:3037] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.516800 22847 raft_consensus.cc:1261] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:33.517174 22977 raft_consensus.cc:1261] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:33.517153 23826 consensus_queue.cc:1035] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:33.518265 23826 consensus_queue.cc:1035] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:33.524485 23861 mvcc.cc:205] Tried to move back new op lower bound from 7023151978556882944 to 7023151977012224000. Current Snapshot: MvccSnapshot[applied={T|T < 7023151978556882944}]
I20240502 07:58:33.536034 23864 mvcc.cc:205] Tried to move back new op lower bound from 7023151978529398784 to 7023151977610457088. Current Snapshot: MvccSnapshot[applied={T|T < 7023151978529398784}]
I20240502 07:58:33.540766 23866 mvcc.cc:205] Tried to move back new op lower bound from 7023151978556882944 to 7023151977012224000. Current Snapshot: MvccSnapshot[applied={T|T < 7023151978556882944}]
INSERT report
rows total: 2000
time total: 103.05 ms
time per row: 0.0515249 ms
I20240502 07:58:33.571610 22435 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_5"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:33.576740 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 88c1176b415c412fb2c824f277e4d3c8 (DEFAULT_TABLE table=rebalance_test_table_5 [id=cd54bf3678754d7291af91c25c402e8d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.576887 22958 tablet_service.cc:1467] Processing CreateTablet for tablet 88c1176b415c412fb2c824f277e4d3c8 (DEFAULT_TABLE table=rebalance_test_table_5 [id=cd54bf3678754d7291af91c25c402e8d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.577032 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 88c1176b415c412fb2c824f277e4d3c8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.577109 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 88c1176b415c412fb2c824f277e4d3c8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.576638 22696 tablet_service.cc:1467] Processing CreateTablet for tablet 88c1176b415c412fb2c824f277e4d3c8 (DEFAULT_TABLE table=rebalance_test_table_5 [id=cd54bf3678754d7291af91c25c402e8d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.577484 22695 tablet_service.cc:1467] Processing CreateTablet for tablet a5374313bbba4b10904e0ea18851c07d (DEFAULT_TABLE table=rebalance_test_table_5 [id=cd54bf3678754d7291af91c25c402e8d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.579274 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 88c1176b415c412fb2c824f277e4d3c8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.579013 23890 tablet_bootstrap.cc:492] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:33.579393 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a5374313bbba4b10904e0ea18851c07d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.579766 22828 tablet_service.cc:1467] Processing CreateTablet for tablet a5374313bbba4b10904e0ea18851c07d (DEFAULT_TABLE table=rebalance_test_table_5 [id=cd54bf3678754d7291af91c25c402e8d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.579881 22566 tablet_service.cc:1467] Processing CreateTablet for tablet a5374313bbba4b10904e0ea18851c07d (DEFAULT_TABLE table=rebalance_test_table_5 [id=cd54bf3678754d7291af91c25c402e8d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.579952 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a5374313bbba4b10904e0ea18851c07d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.580106 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a5374313bbba4b10904e0ea18851c07d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.582474 23890 tablet_bootstrap.cc:654] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.578917 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 55fb5c7475b14e0eaced5b8de1792641 (DEFAULT_TABLE table=rebalance_test_table_5 [id=cd54bf3678754d7291af91c25c402e8d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.582821 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 55fb5c7475b14e0eaced5b8de1792641. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.583431 23890 tablet_bootstrap.cc:492] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:33.583503 23890 ts_tablet_manager.cc:1367] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:58:33.583914 23890 raft_consensus.cc:348] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.583999 23890 raft_consensus.cc:374] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.584024 23890 raft_consensus.cc:729] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.584132 23890 consensus_queue.cc:260] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.584393 23890 ts_tablet_manager.cc:1398] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:33.585027 22958 tablet_service.cc:1467] Processing CreateTablet for tablet 55fb5c7475b14e0eaced5b8de1792641 (DEFAULT_TABLE table=rebalance_test_table_5 [id=cd54bf3678754d7291af91c25c402e8d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.585214 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 55fb5c7475b14e0eaced5b8de1792641. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.586048 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 55fb5c7475b14e0eaced5b8de1792641 (DEFAULT_TABLE table=rebalance_test_table_5 [id=cd54bf3678754d7291af91c25c402e8d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:33.586263 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 55fb5c7475b14e0eaced5b8de1792641. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:33.586690 23890 tablet_bootstrap.cc:492] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:33.587471 23890 tablet_bootstrap.cc:654] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.588544 23890 tablet_bootstrap.cc:492] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:33.588629 23890 ts_tablet_manager.cc:1367] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:33.588773 23892 tablet_bootstrap.cc:492] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:33.589021 23890 raft_consensus.cc:348] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.589103 23890 raft_consensus.cc:374] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.589135 23890 raft_consensus.cc:729] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.589239 23890 consensus_queue.cc:260] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.589592 23892 tablet_bootstrap.cc:654] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.590039 23890 ts_tablet_manager.cc:1398] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:33.590153 23889 tablet_bootstrap.cc:492] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:33.591331 23889 tablet_bootstrap.cc:654] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.591718 23892 tablet_bootstrap.cc:492] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:33.591809 23892 ts_tablet_manager.cc:1367] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:33.592146 23892 raft_consensus.cc:348] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.592267 23892 raft_consensus.cc:374] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.592319 23892 raft_consensus.cc:729] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.592447 23892 consensus_queue.cc:260] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.593209 23892 ts_tablet_manager.cc:1398] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:33.593449 23891 tablet_bootstrap.cc:492] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:33.593794 23889 tablet_bootstrap.cc:492] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:33.593899 23889 ts_tablet_manager.cc:1367] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:33.594090 23891 tablet_bootstrap.cc:654] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.594316 23889 raft_consensus.cc:348] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.594589 23889 raft_consensus.cc:374] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.594663 23889 raft_consensus.cc:729] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.594823 23889 consensus_queue.cc:260] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.594938 23891 tablet_bootstrap.cc:492] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:33.595016 23891 ts_tablet_manager.cc:1367] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:33.595150 23889 ts_tablet_manager.cc:1398] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:33.595301 23889 tablet_bootstrap.cc:492] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:33.595394 23891 raft_consensus.cc:348] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.595479 23891 raft_consensus.cc:374] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.595512 23891 raft_consensus.cc:729] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.595613 23891 consensus_queue.cc:260] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.595803 23891 ts_tablet_manager.cc:1398] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:33.595937 23891 tablet_bootstrap.cc:492] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:33.595994 23889 tablet_bootstrap.cc:654] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.596683 23891 tablet_bootstrap.cc:654] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.596869 23889 tablet_bootstrap.cc:492] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:33.596942 23889 ts_tablet_manager.cc:1367] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:33.597447 23889 raft_consensus.cc:348] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.597539 23889 raft_consensus.cc:374] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.597566 23889 raft_consensus.cc:729] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.597709 23889 consensus_queue.cc:260] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.597810 23891 tablet_bootstrap.cc:492] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:33.597877 23891 ts_tablet_manager.cc:1367] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:33.598017 23889 ts_tablet_manager.cc:1398] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:33.598181 23889 tablet_bootstrap.cc:492] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:33.598287 23891 raft_consensus.cc:348] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.598824 23889 tablet_bootstrap.cc:654] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.599033 23891 raft_consensus.cc:374] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.599120 23891 raft_consensus.cc:729] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.599295 23891 consensus_queue.cc:260] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.599581 23891 ts_tablet_manager.cc:1398] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:33.599625 23889 tablet_bootstrap.cc:492] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:33.599694 23889 ts_tablet_manager.cc:1367] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:33.599732 23891 tablet_bootstrap.cc:492] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:33.599984 23889 raft_consensus.cc:348] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.600052 23889 raft_consensus.cc:374] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.600080 23889 raft_consensus.cc:729] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.600178 23889 consensus_queue.cc:260] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.600384 23889 ts_tablet_manager.cc:1398] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:33.600462 23891 tablet_bootstrap.cc:654] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:33.601382 23891 tablet_bootstrap.cc:492] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:33.601454 23891 ts_tablet_manager.cc:1367] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:33.601848 23891 raft_consensus.cc:348] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.601948 23891 raft_consensus.cc:374] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:33.601992 23891 raft_consensus.cc:729] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:33.602113 23891 consensus_queue.cc:260] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.602728 23891 ts_tablet_manager.cc:1398] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:33.626549 23721 raft_consensus.cc:482] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:33.626659 23721 raft_consensus.cc:504] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.627003 23721 leader_election.cc:288] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:33.627324 22977 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "55fb5c7475b14e0eaced5b8de1792641" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:33.627355 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "55fb5c7475b14e0eaced5b8de1792641" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:33.627444 22977 raft_consensus.cc:2454] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:33.627465 22847 raft_consensus.cc:2454] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:33.627671 22651 leader_election.cc:302] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:33.627804 23721 raft_consensus.cc:2781] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:33.627854 23721 raft_consensus.cc:482] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:33.627897 23721 raft_consensus.cc:3037] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.628696 23721 raft_consensus.cc:504] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.629084 23721 leader_election.cc:288] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:33.629318 22977 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "55fb5c7475b14e0eaced5b8de1792641" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:33.629390 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "55fb5c7475b14e0eaced5b8de1792641" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:33.629426 22977 raft_consensus.cc:3037] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.629498 22847 raft_consensus.cc:3037] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.630172 22977 raft_consensus.cc:2454] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:33.630219 22847 raft_consensus.cc:2454] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:33.630473 22651 leader_election.cc:302] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:33.630847 23721 raft_consensus.cc:2781] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:33.631007 23721 raft_consensus.cc:686] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:33.631139 23721 consensus_queue.cc:237] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.632959 22435 catalog_manager.cc:5484] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:33.684276 23805 raft_consensus.cc:482] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:33.684377 23805 raft_consensus.cc:504] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.684768 23805 leader_election.cc:288] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:33.685086 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "88c1176b415c412fb2c824f277e4d3c8" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:33.685149 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "88c1176b415c412fb2c824f277e4d3c8" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:33.685225 22717 raft_consensus.cc:2454] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:33.685267 22847 raft_consensus.cc:2454] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:33.685508 22915 leader_election.cc:302] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:33.685655 23805 raft_consensus.cc:2781] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:33.685708 23805 raft_consensus.cc:482] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:33.685770 23805 raft_consensus.cc:3037] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.686482 23805 raft_consensus.cc:504] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.686744 23805 leader_election.cc:288] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:33.686985 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "88c1176b415c412fb2c824f277e4d3c8" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:33.687029 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "88c1176b415c412fb2c824f277e4d3c8" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:33.687086 22717 raft_consensus.cc:3037] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.687114 22847 raft_consensus.cc:3037] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.687789 22847 raft_consensus.cc:2454] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:33.688050 22915 leader_election.cc:302] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:33.688184 23805 raft_consensus.cc:2781] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:33.688361 23805 raft_consensus.cc:686] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:33.688484 23805 consensus_queue.cc:237] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:33.689949 22435 catalog_manager.cc:5484] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:33.713505 22717 raft_consensus.cc:2454] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:33.939245 23879 raft_consensus.cc:482] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:33.939370 23879 raft_consensus.cc:504] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.939724 23879 leader_election.cc:288] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:33.940035 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a5374313bbba4b10904e0ea18851c07d" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:33.940057 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a5374313bbba4b10904e0ea18851c07d" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:33.940153 22847 raft_consensus.cc:2454] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:33.940183 22717 raft_consensus.cc:2454] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:33.940433 22522 leader_election.cc:302] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:33.940583 23879 raft_consensus.cc:2781] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:33.940637 23879 raft_consensus.cc:482] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:33.940708 23879 raft_consensus.cc:3037] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.941419 23879 raft_consensus.cc:504] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.941677 23879 leader_election.cc:288] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:33.942024 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a5374313bbba4b10904e0ea18851c07d" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:33.942142 22847 raft_consensus.cc:3037] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.942178 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a5374313bbba4b10904e0ea18851c07d" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:33.942297 22717 raft_consensus.cc:3037] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:33.943053 22847 raft_consensus.cc:2454] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:33.943192 22717 raft_consensus.cc:2454] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:33.943316 22522 leader_election.cc:302] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:33.943454 23879 raft_consensus.cc:2781] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:33.943532 23879 raft_consensus.cc:686] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:33.943682 23879 consensus_queue.cc:237] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:33.945122 22435 catalog_manager.cc:5484] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:34.022630 23721 consensus_queue.cc:1035] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:34.025282 23908 consensus_queue.cc:1035] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:58:34.128257 23910 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:34.128461 23910 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:34.131494 23910 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_5'
I20240502 07:58:34.204810 22717 raft_consensus.cc:1261] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:34.205348 23879 consensus_queue.cc:1035] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:34.208437 22847 raft_consensus.cc:1261] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:34.209175 22847 raft_consensus.cc:1261] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:34.209395 22715 raft_consensus.cc:1261] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:34.209574 23879 consensus_queue.cc:1035] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:34.209550 23942 consensus_queue.cc:1035] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:34.209921 23805 consensus_queue.cc:1035] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:34.218672 23862 mvcc.cc:205] Tried to move back new op lower bound from 7023151981392568320 to 7023151979269230592. Current Snapshot: MvccSnapshot[applied={T|T < 7023151981392568320}]
INSERT report
rows total: 2000
time total: 104.957 ms
time per row: 0.0524786 ms
I20240502 07:58:34.273378 22435 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_6"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:34.278201 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 1a5ce428e35849f7a3f2148f02fede71 (DEFAULT_TABLE table=rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.278492 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1a5ce428e35849f7a3f2148f02fede71. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.278561 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 1a5ce428e35849f7a3f2148f02fede71 (DEFAULT_TABLE table=rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.278788 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1a5ce428e35849f7a3f2148f02fede71. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.278815 22958 tablet_service.cc:1467] Processing CreateTablet for tablet 1a5ce428e35849f7a3f2148f02fede71 (DEFAULT_TABLE table=rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.279018 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1a5ce428e35849f7a3f2148f02fede71. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.280884 23964 tablet_bootstrap.cc:492] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:34.280858 22564 tablet_service.cc:1467] Processing CreateTablet for tablet ddebb5e20e134841886be53f8f0d5e02 (DEFAULT_TABLE table=rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.280864 22696 tablet_service.cc:1467] Processing CreateTablet for tablet 5c36c7654df64dbebdf72ddf403771ab (DEFAULT_TABLE table=rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.281069 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ddebb5e20e134841886be53f8f0d5e02. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.281075 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5c36c7654df64dbebdf72ddf403771ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.281740 23964 tablet_bootstrap.cc:654] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.283354 22958 tablet_service.cc:1467] Processing CreateTablet for tablet ddebb5e20e134841886be53f8f0d5e02 (DEFAULT_TABLE table=rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.283408 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 5c36c7654df64dbebdf72ddf403771ab (DEFAULT_TABLE table=rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.283560 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ddebb5e20e134841886be53f8f0d5e02. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.283531 22827 tablet_service.cc:1467] Processing CreateTablet for tablet ddebb5e20e134841886be53f8f0d5e02 (DEFAULT_TABLE table=rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.283619 23962 tablet_bootstrap.cc:492] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:34.283694 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5c36c7654df64dbebdf72ddf403771ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.283771 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ddebb5e20e134841886be53f8f0d5e02. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.283864 23963 tablet_bootstrap.cc:492] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:34.285466 23962 tablet_bootstrap.cc:654] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.285797 23966 tablet_bootstrap.cc:492] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:34.284631 23963 tablet_bootstrap.cc:654] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.285923 23964 tablet_bootstrap.cc:492] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:34.286015 23964 ts_tablet_manager.cc:1367] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:58:34.286553 23964 raft_consensus.cc:348] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.286794 23964 raft_consensus.cc:374] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.286871 23963 tablet_bootstrap.cc:492] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:34.286880 23964 raft_consensus.cc:729] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.286906 23966 tablet_bootstrap.cc:654] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.286949 23963 ts_tablet_manager.cc:1367] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:34.286969 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 5c36c7654df64dbebdf72ddf403771ab (DEFAULT_TABLE table=rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.287030 23964 consensus_queue.cc:260] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.287178 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5c36c7654df64dbebdf72ddf403771ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.287376 23964 ts_tablet_manager.cc:1398] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:34.287343 23963 raft_consensus.cc:348] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.287425 23963 raft_consensus.cc:374] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.287453 23963 raft_consensus.cc:729] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.287508 23964 tablet_bootstrap.cc:492] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:34.287570 23963 consensus_queue.cc:260] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.287811 23963 ts_tablet_manager.cc:1398] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:34.288034 23963 tablet_bootstrap.cc:492] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:34.288329 23964 tablet_bootstrap.cc:654] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.288915 23963 tablet_bootstrap.cc:654] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.289274 23966 tablet_bootstrap.cc:492] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:34.289382 23966 ts_tablet_manager.cc:1367] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:34.289819 23966 raft_consensus.cc:348] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:34.289917 23966 raft_consensus.cc:374] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.289966 23966 raft_consensus.cc:729] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.290093 23966 consensus_queue.cc:260] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:34.290160 23963 tablet_bootstrap.cc:492] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:34.290221 23963 ts_tablet_manager.cc:1367] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:34.290367 23966 ts_tablet_manager.cc:1398] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:34.290784 23963 raft_consensus.cc:348] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.290876 23963 raft_consensus.cc:374] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.290904 23963 raft_consensus.cc:729] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.291002 23963 consensus_queue.cc:260] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.291301 23963 ts_tablet_manager.cc:1398] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:34.291451 23963 tablet_bootstrap.cc:492] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:34.292194 23963 tablet_bootstrap.cc:654] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.292840 23962 tablet_bootstrap.cc:492] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:34.292927 23962 ts_tablet_manager.cc:1367] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.009s user 0.002s sys 0.000s
I20240502 07:58:34.293195 23963 tablet_bootstrap.cc:492] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:34.293260 23963 ts_tablet_manager.cc:1367] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:34.293330 23962 raft_consensus.cc:348] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.293421 23962 raft_consensus.cc:374] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.293447 23962 raft_consensus.cc:729] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.293578 23962 consensus_queue.cc:260] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.293576 23963 raft_consensus.cc:348] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:34.293648 23963 raft_consensus.cc:374] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.293665 23963 raft_consensus.cc:729] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.293769 23963 consensus_queue.cc:260] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:34.293818 23962 ts_tablet_manager.cc:1398] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:34.293954 23962 tablet_bootstrap.cc:492] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:34.294023 23963 ts_tablet_manager.cc:1398] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:34.294641 23962 tablet_bootstrap.cc:654] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.295542 23962 tablet_bootstrap.cc:492] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:34.295603 23962 ts_tablet_manager.cc:1367] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:34.295897 23962 raft_consensus.cc:348] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:34.295979 23962 raft_consensus.cc:374] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.296020 23962 raft_consensus.cc:729] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.296118 23962 consensus_queue.cc:260] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:34.296317 23962 ts_tablet_manager.cc:1398] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:34.296449 23962 tablet_bootstrap.cc:492] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:34.297173 23962 tablet_bootstrap.cc:654] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.297170 23964 tablet_bootstrap.cc:492] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:34.297242 23964 ts_tablet_manager.cc:1367] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.010s user 0.001s sys 0.000s
I20240502 07:58:34.297689 23964 raft_consensus.cc:348] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.297785 23964 raft_consensus.cc:374] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.297819 23964 raft_consensus.cc:729] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.297943 23964 consensus_queue.cc:260] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.298202 23964 ts_tablet_manager.cc:1398] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:34.298378 23962 tablet_bootstrap.cc:492] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:34.298457 23962 ts_tablet_manager.cc:1367] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:34.298779 23962 raft_consensus.cc:348] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.298847 23962 raft_consensus.cc:374] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.298866 23962 raft_consensus.cc:729] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.298960 23962 consensus_queue.cc:260] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.299142 23962 ts_tablet_manager.cc:1398] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:34.300630 23942 raft_consensus.cc:482] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:34.300734 23942 raft_consensus.cc:504] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.301080 23942 leader_election.cc:288] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:34.301306 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a5ce428e35849f7a3f2148f02fede71" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:34.301409 22586 raft_consensus.cc:2454] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:34.301430 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a5ce428e35849f7a3f2148f02fede71" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:34.301539 22717 raft_consensus.cc:2454] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:34.301671 22915 leader_election.cc:302] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:34.301820 23942 raft_consensus.cc:2781] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:34.301891 23942 raft_consensus.cc:482] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:34.301939 23942 raft_consensus.cc:3037] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.302722 23942 raft_consensus.cc:504] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.303031 23942 leader_election.cc:288] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:34.303273 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a5ce428e35849f7a3f2148f02fede71" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:34.303318 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a5ce428e35849f7a3f2148f02fede71" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:34.303362 22586 raft_consensus.cc:3037] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.303395 22717 raft_consensus.cc:3037] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.303952 22586 raft_consensus.cc:2454] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:34.304008 22717 raft_consensus.cc:2454] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:34.304265 22912 leader_election.cc:302] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:34.304440 23942 raft_consensus.cc:2781] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:34.304517 23942 raft_consensus.cc:686] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:34.304618 23942 consensus_queue.cc:237] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.305958 22439 catalog_manager.cc:5484] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:34.365700 23879 raft_consensus.cc:482] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:34.365810 23879 raft_consensus.cc:504] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:34.366190 23879 leader_election.cc:288] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:34.366528 22977 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddebb5e20e134841886be53f8f0d5e02" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:34.366616 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddebb5e20e134841886be53f8f0d5e02" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:34.366673 22977 raft_consensus.cc:2454] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:34.366783 22848 raft_consensus.cc:2454] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:34.366966 22520 leader_election.cc:302] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:34.367148 23879 raft_consensus.cc:2781] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:34.367205 23879 raft_consensus.cc:482] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:34.367265 23879 raft_consensus.cc:3037] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.368048 23879 raft_consensus.cc:504] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:34.368381 23879 leader_election.cc:288] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:34.368633 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddebb5e20e134841886be53f8f0d5e02" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:34.368702 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddebb5e20e134841886be53f8f0d5e02" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:34.368780 22979 raft_consensus.cc:3037] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.368810 22848 raft_consensus.cc:3037] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.369516 22848 raft_consensus.cc:2454] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:34.369519 22979 raft_consensus.cc:2454] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:34.369758 22522 leader_election.cc:302] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:34.369939 23879 raft_consensus.cc:2781] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:34.370040 23879 raft_consensus.cc:686] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:34.370180 23879 consensus_queue.cc:237] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:34.371742 22439 catalog_manager.cc:5484] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:34.458133 23942 raft_consensus.cc:482] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:34.458237 23942 raft_consensus.cc:504] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.458617 23942 leader_election.cc:288] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:34.458977 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c36c7654df64dbebdf72ddf403771ab" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:34.459026 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c36c7654df64dbebdf72ddf403771ab" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:34.459116 22586 raft_consensus.cc:2454] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:34.459177 22717 raft_consensus.cc:2454] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:34.459353 22915 leader_election.cc:302] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:34.459482 23942 raft_consensus.cc:2781] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:34.459530 23942 raft_consensus.cc:482] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:34.459573 23942 raft_consensus.cc:3037] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.460333 23942 raft_consensus.cc:504] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.460613 23942 leader_election.cc:288] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:34.460851 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c36c7654df64dbebdf72ddf403771ab" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:34.460892 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c36c7654df64dbebdf72ddf403771ab" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:34.460943 22586 raft_consensus.cc:3037] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.460969 22717 raft_consensus.cc:3037] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.461659 22586 raft_consensus.cc:2454] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:34.461761 22717 raft_consensus.cc:2454] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:34.461917 22915 leader_election.cc:302] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:34.462055 23942 raft_consensus.cc:2781] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:34.462188 23942 raft_consensus.cc:686] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:34.462321 23942 consensus_queue.cc:237] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.463940 22439 catalog_manager.cc:5484] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:34.567127 23978 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:34.567333 23978 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:34.570060 23978 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_6'
I20240502 07:58:34.649441 22717 raft_consensus.cc:1261] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:34.649472 22586 raft_consensus.cc:1261] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:34.649905 23942 consensus_queue.cc:1035] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:34.650204 23805 consensus_queue.cc:1035] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:34.655086 22979 raft_consensus.cc:1261] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:34.655359 22584 raft_consensus.cc:1261] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:34.655670 22848 raft_consensus.cc:1261] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:34.655680 23942 consensus_queue.cc:1035] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:34.655989 22715 raft_consensus.cc:1261] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:34.656304 23879 consensus_queue.cc:1035] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:34.656862 23879 consensus_queue.cc:1035] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:34.657459 23942 consensus_queue.cc:1035] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 109.195 ms
time per row: 0.0545978 ms
I20240502 07:58:34.715755 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_7"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:34.720793 22958 tablet_service.cc:1467] Processing CreateTablet for tablet 9ca2e5b5a0d24bcc9d1b69f998a0d928 (DEFAULT_TABLE table=rebalance_test_table_7 [id=b66f182129cf49c0a53671ec66e4a6e2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.720846 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 9ca2e5b5a0d24bcc9d1b69f998a0d928 (DEFAULT_TABLE table=rebalance_test_table_7 [id=b66f182129cf49c0a53671ec66e4a6e2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.721066 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9ca2e5b5a0d24bcc9d1b69f998a0d928. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.721092 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9ca2e5b5a0d24bcc9d1b69f998a0d928. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.721139 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 6dbf36c0f044466ab13a53bfe0d6cb53 (DEFAULT_TABLE table=rebalance_test_table_7 [id=b66f182129cf49c0a53671ec66e4a6e2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.721372 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6dbf36c0f044466ab13a53bfe0d6cb53. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.722782 23966 tablet_bootstrap.cc:492] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:34.723152 23963 tablet_bootstrap.cc:492] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:34.723436 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 6dbf36c0f044466ab13a53bfe0d6cb53 (DEFAULT_TABLE table=rebalance_test_table_7 [id=b66f182129cf49c0a53671ec66e4a6e2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.723543 23966 tablet_bootstrap.cc:654] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.723634 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6dbf36c0f044466ab13a53bfe0d6cb53. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.723901 23963 tablet_bootstrap.cc:654] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.724588 23966 tablet_bootstrap.cc:492] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:34.724646 23966 ts_tablet_manager.cc:1367] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:34.724961 23963 tablet_bootstrap.cc:492] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:34.725023 23963 ts_tablet_manager.cc:1367] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:34.724985 23966 raft_consensus.cc:348] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:34.725054 23966 raft_consensus.cc:374] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.725072 23966 raft_consensus.cc:729] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.725169 23966 consensus_queue.cc:260] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:34.725441 23966 ts_tablet_manager.cc:1398] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:34.725406 23963 raft_consensus.cc:348] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:34.725487 23963 raft_consensus.cc:374] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.725524 23963 raft_consensus.cc:729] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.725574 23966 tablet_bootstrap.cc:492] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:34.725616 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 9ca2e5b5a0d24bcc9d1b69f998a0d928 (DEFAULT_TABLE table=rebalance_test_table_7 [id=b66f182129cf49c0a53671ec66e4a6e2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.725637 23963 consensus_queue.cc:260] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:34.725804 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9ca2e5b5a0d24bcc9d1b69f998a0d928. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.725865 23963 ts_tablet_manager.cc:1398] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:34.726163 22958 tablet_service.cc:1467] Processing CreateTablet for tablet abfc26b86b17464fa7cf0b13ab73861c (DEFAULT_TABLE table=rebalance_test_table_7 [id=b66f182129cf49c0a53671ec66e4a6e2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.726354 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet abfc26b86b17464fa7cf0b13ab73861c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.726473 23966 tablet_bootstrap.cc:654] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.727916 23966 tablet_bootstrap.cc:492] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:34.727936 23962 tablet_bootstrap.cc:492] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:34.727984 23966 ts_tablet_manager.cc:1367] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:34.728078 23963 tablet_bootstrap.cc:492] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:34.728323 23966 raft_consensus.cc:348] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.728385 23966 raft_consensus.cc:374] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.728402 23966 raft_consensus.cc:729] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.728489 23966 consensus_queue.cc:260] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.728574 23943 raft_consensus.cc:482] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:34.728616 23943 raft_consensus.cc:504] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.728837 23962 tablet_bootstrap.cc:654] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.728837 23963 tablet_bootstrap.cc:654] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.728957 23943 leader_election.cc:288] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:34.729810 23962 tablet_bootstrap.cc:492] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:34.729882 23962 ts_tablet_manager.cc:1367] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:34.730022 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6dbf36c0f044466ab13a53bfe0d6cb53" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:34.730264 23962 raft_consensus.cc:348] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.730357 23962 raft_consensus.cc:374] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.730387 23962 raft_consensus.cc:729] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.730489 23962 consensus_queue.cc:260] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
W20240502 07:58:34.730540 22784 leader_election.cc:341] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): Illegal state: must be running to vote when last-logged opid is not known
I20240502 07:58:34.730816 23966 ts_tablet_manager.cc:1398] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.003s user 0.000s sys 0.001s
I20240502 07:58:34.730821 23962 ts_tablet_manager.cc:1398] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:34.730969 23962 tablet_bootstrap.cc:492] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:34.731395 22696 tablet_service.cc:1467] Processing CreateTablet for tablet 6dbf36c0f044466ab13a53bfe0d6cb53 (DEFAULT_TABLE table=rebalance_test_table_7 [id=b66f182129cf49c0a53671ec66e4a6e2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.731642 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6dbf36c0f044466ab13a53bfe0d6cb53. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.731974 23962 tablet_bootstrap.cc:654] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.733742 23964 tablet_bootstrap.cc:492] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:34.733886 22697 tablet_service.cc:1467] Processing CreateTablet for tablet abfc26b86b17464fa7cf0b13ab73861c (DEFAULT_TABLE table=rebalance_test_table_7 [id=b66f182129cf49c0a53671ec66e4a6e2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.734062 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet abfc26b86b17464fa7cf0b13ab73861c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.734455 23964 tablet_bootstrap.cc:654] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.735337 23964 tablet_bootstrap.cc:492] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:34.735415 23964 ts_tablet_manager.cc:1367] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:34.735776 23964 raft_consensus.cc:348] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.735853 23964 raft_consensus.cc:374] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.735880 23964 raft_consensus.cc:729] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.735982 23964 consensus_queue.cc:260] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.736241 23964 ts_tablet_manager.cc:1398] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:34.736451 23964 tablet_bootstrap.cc:492] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:34.737200 23964 tablet_bootstrap.cc:654] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.737257 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6dbf36c0f044466ab13a53bfe0d6cb53" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:34.737370 22717 raft_consensus.cc:2454] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:34.737617 22781 leader_election.cc:302] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters: 8bea0fe8d8044a2685f92f7a10c9cf85
I20240502 07:58:34.737838 23943 raft_consensus.cc:2781] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:34.737879 23943 raft_consensus.cc:482] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:34.737915 23943 raft_consensus.cc:3037] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.738080 23964 tablet_bootstrap.cc:492] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:34.738137 23964 ts_tablet_manager.cc:1367] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:34.738484 23964 raft_consensus.cc:348] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:34.738572 23964 raft_consensus.cc:374] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.738606 23964 raft_consensus.cc:729] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.738718 23943 raft_consensus.cc:504] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.738716 23964 consensus_queue.cc:260] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:34.738955 23964 ts_tablet_manager.cc:1398] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:34.739034 23943 leader_election.cc:288] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:34.739683 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6dbf36c0f044466ab13a53bfe0d6cb53" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:34.739781 22586 raft_consensus.cc:3037] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.740599 22586 raft_consensus.cc:2454] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:34.740782 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6dbf36c0f044466ab13a53bfe0d6cb53" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:34.740875 22717 raft_consensus.cc:3037] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.741698 22717 raft_consensus.cc:2454] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:34.741974 22784 leader_election.cc:302] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:34.742595 23943 raft_consensus.cc:2781] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:34.742748 23943 raft_consensus.cc:686] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:34.742878 23943 consensus_queue.cc:237] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:34.743860 22827 tablet_service.cc:1467] Processing CreateTablet for tablet abfc26b86b17464fa7cf0b13ab73861c (DEFAULT_TABLE table=rebalance_test_table_7 [id=b66f182129cf49c0a53671ec66e4a6e2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:34.744073 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet abfc26b86b17464fa7cf0b13ab73861c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:34.744264 22443 catalog_manager.cc:5484] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:34.745695 23966 tablet_bootstrap.cc:492] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:34.746261 23962 tablet_bootstrap.cc:492] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:34.746340 23962 ts_tablet_manager.cc:1367] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.015s user 0.000s sys 0.002s
I20240502 07:58:34.746435 23963 tablet_bootstrap.cc:492] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:34.746510 23963 ts_tablet_manager.cc:1367] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.018s user 0.000s sys 0.001s
I20240502 07:58:34.746549 23966 tablet_bootstrap.cc:654] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:34.746834 23963 raft_consensus.cc:348] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:34.746902 23963 raft_consensus.cc:374] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.746845 23962 raft_consensus.cc:348] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:34.746922 23963 raft_consensus.cc:729] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.746932 23962 raft_consensus.cc:374] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.746963 23962 raft_consensus.cc:729] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.747020 23963 consensus_queue.cc:260] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:34.747100 23962 consensus_queue.cc:260] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:34.747288 23963 ts_tablet_manager.cc:1398] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:34.747370 23962 ts_tablet_manager.cc:1398] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:34.747887 23966 tablet_bootstrap.cc:492] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:34.747968 23966 ts_tablet_manager.cc:1367] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:34.748370 23966 raft_consensus.cc:348] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:34.748471 23966 raft_consensus.cc:374] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:34.748509 23966 raft_consensus.cc:729] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:34.748648 23966 consensus_queue.cc:260] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:34.748936 23966 ts_tablet_manager.cc:1398] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:34.827608 23943 raft_consensus.cc:482] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:34.827719 23943 raft_consensus.cc:504] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:34.828076 23943 leader_election.cc:288] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:34.828373 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "abfc26b86b17464fa7cf0b13ab73861c" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:34.828428 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "abfc26b86b17464fa7cf0b13ab73861c" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:34.828510 22717 raft_consensus.cc:2454] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:34.828549 22979 raft_consensus.cc:2454] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:34.828783 22781 leader_election.cc:302] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:34.828991 23943 raft_consensus.cc:2781] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:34.829036 23943 raft_consensus.cc:482] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:34.829078 23943 raft_consensus.cc:3037] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.829921 23943 raft_consensus.cc:504] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:34.830214 23943 leader_election.cc:288] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:34.830536 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "abfc26b86b17464fa7cf0b13ab73861c" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:34.830538 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "abfc26b86b17464fa7cf0b13ab73861c" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:34.830621 22979 raft_consensus.cc:3037] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.830632 22717 raft_consensus.cc:3037] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.831446 22979 raft_consensus.cc:2454] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:34.831501 22717 raft_consensus.cc:2454] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:34.831719 22782 leader_election.cc:302] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:34.831903 23943 raft_consensus.cc:2781] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:34.831997 23943 raft_consensus.cc:686] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:34.832134 23943 consensus_queue.cc:237] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:34.833704 22443 catalog_manager.cc:5484] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:34.835176 24007 raft_consensus.cc:482] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:34.835296 24007 raft_consensus.cc:504] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:34.835673 24007 leader_election.cc:288] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:34.835929 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ca2e5b5a0d24bcc9d1b69f998a0d928" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:34.836000 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ca2e5b5a0d24bcc9d1b69f998a0d928" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:34.836036 22979 raft_consensus.cc:2454] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:34.836094 22848 raft_consensus.cc:2454] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:34.836266 22520 leader_election.cc:302] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:34.836416 24007 raft_consensus.cc:2781] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:34.836477 24007 raft_consensus.cc:482] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:34.836519 24007 raft_consensus.cc:3037] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.837249 24007 raft_consensus.cc:504] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:34.837564 24007 leader_election.cc:288] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:34.837865 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ca2e5b5a0d24bcc9d1b69f998a0d928" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:34.837896 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9ca2e5b5a0d24bcc9d1b69f998a0d928" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:34.837956 22979 raft_consensus.cc:3037] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.837970 22848 raft_consensus.cc:3037] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:34.838557 22979 raft_consensus.cc:2454] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:34.838560 22848 raft_consensus.cc:2454] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:34.838816 22522 leader_election.cc:302] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:34.838965 24007 raft_consensus.cc:2781] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:34.839042 24007 raft_consensus.cc:686] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:34.839167 24007 consensus_queue.cc:237] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:34.840606 22443 catalog_manager.cc:5484] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } }
W20240502 07:58:34.939136 24034 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:34.939323 24034 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:34.942054 24034 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_7'
I20240502 07:58:35.016204 22979 raft_consensus.cc:1261] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.016275 22717 raft_consensus.cc:1261] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.017014 24029 consensus_queue.cc:1035] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:35.017333 22586 raft_consensus.cc:1261] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.018043 23943 consensus_queue.cc:1035] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:35.018321 23943 consensus_queue.cc:1035] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:35.020387 22717 raft_consensus.cc:1261] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.021967 22977 raft_consensus.cc:1261] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.022161 22848 raft_consensus.cc:1261] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.022322 24029 consensus_queue.cc:1035] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:35.022410 24007 consensus_queue.cc:1035] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:35.027443 24007 consensus_queue.cc:1035] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 101.495 ms
time per row: 0.0507473 ms
I20240502 07:58:35.079119 22437 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_8"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:35.083563 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 5124636392c74f56ad54f1ab91b9f7b0 (DEFAULT_TABLE table=rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.083822 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5124636392c74f56ad54f1ab91b9f7b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.084266 22958 tablet_service.cc:1467] Processing CreateTablet for tablet 73ad3c23d96b43b1894f767ad5db1e2f (DEFAULT_TABLE table=rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.084336 22959 tablet_service.cc:1467] Processing CreateTablet for tablet b22f8e50a7d24dbdab665e47120602ce (DEFAULT_TABLE table=rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.084496 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 73ad3c23d96b43b1894f767ad5db1e2f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.084827 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b22f8e50a7d24dbdab665e47120602ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.084846 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 5124636392c74f56ad54f1ab91b9f7b0 (DEFAULT_TABLE table=rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.085038 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5124636392c74f56ad54f1ab91b9f7b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.085851 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 5124636392c74f56ad54f1ab91b9f7b0 (DEFAULT_TABLE table=rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.086048 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5124636392c74f56ad54f1ab91b9f7b0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.086736 23966 tablet_bootstrap.cc:492] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:35.086831 22828 tablet_service.cc:1467] Processing CreateTablet for tablet b22f8e50a7d24dbdab665e47120602ce (DEFAULT_TABLE table=rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.087010 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b22f8e50a7d24dbdab665e47120602ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.087524 23966 tablet_bootstrap.cc:654] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.087922 23964 tablet_bootstrap.cc:492] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:35.088070 22696 tablet_service.cc:1467] Processing CreateTablet for tablet 73ad3c23d96b43b1894f767ad5db1e2f (DEFAULT_TABLE table=rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.088279 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 73ad3c23d96b43b1894f767ad5db1e2f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.088778 23964 tablet_bootstrap.cc:654] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.088860 23963 tablet_bootstrap.cc:492] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:35.089553 23963 tablet_bootstrap.cc:654] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.089797 23964 tablet_bootstrap.cc:492] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:35.089867 23964 ts_tablet_manager.cc:1367] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:35.090107 23962 tablet_bootstrap.cc:492] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:35.090237 23964 raft_consensus.cc:348] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.090339 23964 raft_consensus.cc:374] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.090369 23964 raft_consensus.cc:729] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.090468 23963 tablet_bootstrap.cc:492] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:35.090544 23963 ts_tablet_manager.cc:1367] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:35.090498 23964 consensus_queue.cc:260] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.090795 23964 ts_tablet_manager.cc:1398] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:35.090886 23963 raft_consensus.cc:348] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.090924 23962 tablet_bootstrap.cc:654] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.090952 23963 raft_consensus.cc:374] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.090950 23964 tablet_bootstrap.cc:492] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:35.090975 23963 raft_consensus.cc:729] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.091079 23963 consensus_queue.cc:260] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.091362 23963 ts_tablet_manager.cc:1398] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:35.091516 23963 tablet_bootstrap.cc:492] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:35.091789 23964 tablet_bootstrap.cc:654] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.091852 22564 tablet_service.cc:1467] Processing CreateTablet for tablet b22f8e50a7d24dbdab665e47120602ce (DEFAULT_TABLE table=rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.091979 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 73ad3c23d96b43b1894f767ad5db1e2f (DEFAULT_TABLE table=rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.092114 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b22f8e50a7d24dbdab665e47120602ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.092535 23963 tablet_bootstrap.cc:654] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.093012 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 73ad3c23d96b43b1894f767ad5db1e2f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.093247 23962 tablet_bootstrap.cc:492] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:35.093317 23962 ts_tablet_manager.cc:1367] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:35.093648 23962 raft_consensus.cc:348] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.093724 23962 raft_consensus.cc:374] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.093757 23962 raft_consensus.cc:729] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.093858 23962 consensus_queue.cc:260] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.094082 23962 ts_tablet_manager.cc:1398] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:35.094201 23962 tablet_bootstrap.cc:492] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:35.094806 23963 tablet_bootstrap.cc:492] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:35.094871 23963 ts_tablet_manager.cc:1367] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240502 07:58:35.095199 23963 raft_consensus.cc:348] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.095301 23963 raft_consensus.cc:374] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.095331 23963 raft_consensus.cc:729] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.095432 23963 consensus_queue.cc:260] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.095489 23966 tablet_bootstrap.cc:492] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:35.095585 23966 ts_tablet_manager.cc:1367] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.009s user 0.001s sys 0.001s
I20240502 07:58:35.095686 23963 ts_tablet_manager.cc:1398] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:35.095963 23966 raft_consensus.cc:348] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.096052 23966 raft_consensus.cc:374] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.096081 23966 raft_consensus.cc:729] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.096237 23962 tablet_bootstrap.cc:654] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.096199 23966 consensus_queue.cc:260] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.096472 23966 ts_tablet_manager.cc:1398] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:35.096649 23966 tablet_bootstrap.cc:492] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:35.097267 23964 tablet_bootstrap.cc:492] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:35.097340 23964 ts_tablet_manager.cc:1367] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.002s
I20240502 07:58:35.097472 23966 tablet_bootstrap.cc:654] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.097754 23964 raft_consensus.cc:348] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.097839 23964 raft_consensus.cc:374] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.097875 23964 raft_consensus.cc:729] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.098006 23964 consensus_queue.cc:260] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.098234 23964 ts_tablet_manager.cc:1398] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:35.098932 23962 tablet_bootstrap.cc:492] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:35.099002 23962 ts_tablet_manager.cc:1367] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.001s
I20240502 07:58:35.099399 23962 raft_consensus.cc:348] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.099505 23962 raft_consensus.cc:374] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.099542 23962 raft_consensus.cc:729] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.099571 23966 tablet_bootstrap.cc:492] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:35.099642 23966 ts_tablet_manager.cc:1367] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:35.099664 23962 consensus_queue.cc:260] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.099918 23962 ts_tablet_manager.cc:1398] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:35.100009 23966 raft_consensus.cc:348] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.100097 23966 raft_consensus.cc:374] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.100128 23966 raft_consensus.cc:729] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.100207 23962 tablet_bootstrap.cc:492] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:35.100247 23966 consensus_queue.cc:260] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.100500 23966 ts_tablet_manager.cc:1398] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:35.100940 23962 tablet_bootstrap.cc:654] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.101895 23962 tablet_bootstrap.cc:492] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:35.101990 23962 ts_tablet_manager.cc:1367] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.001s
I20240502 07:58:35.102427 23962 raft_consensus.cc:348] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.102520 23962 raft_consensus.cc:374] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.102555 23962 raft_consensus.cc:729] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.102663 23962 consensus_queue.cc:260] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.102926 23962 ts_tablet_manager.cc:1398] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:35.118126 24059 raft_consensus.cc:482] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:35.118203 24059 raft_consensus.cc:504] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.118558 24059 leader_election.cc:288] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:35.118806 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5124636392c74f56ad54f1ab91b9f7b0" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:35.118825 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5124636392c74f56ad54f1ab91b9f7b0" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:35.118916 22586 raft_consensus.cc:2454] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:35.118932 22715 raft_consensus.cc:2454] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:35.119151 22781 leader_election.cc:302] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:35.119298 24059 raft_consensus.cc:2781] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:35.119346 24059 raft_consensus.cc:482] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:35.119374 24059 raft_consensus.cc:3037] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.120091 24059 raft_consensus.cc:504] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.120398 24059 leader_election.cc:288] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:35.120638 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5124636392c74f56ad54f1ab91b9f7b0" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:35.120707 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5124636392c74f56ad54f1ab91b9f7b0" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:35.120787 22586 raft_consensus.cc:3037] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.120787 22715 raft_consensus.cc:3037] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.121626 22715 raft_consensus.cc:2454] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:35.121626 22586 raft_consensus.cc:2454] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:35.121865 22781 leader_election.cc:302] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:35.122046 24059 raft_consensus.cc:2781] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:35.122123 24059 raft_consensus.cc:686] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:35.122258 24059 consensus_queue.cc:237] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.123718 22442 catalog_manager.cc:5484] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:35.224304 24015 raft_consensus.cc:482] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:35.224424 24015 raft_consensus.cc:504] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.224813 24015 leader_election.cc:288] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:35.225109 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b22f8e50a7d24dbdab665e47120602ce" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:35.225111 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b22f8e50a7d24dbdab665e47120602ce" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:35.225229 22586 raft_consensus.cc:2454] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:35.225235 22847 raft_consensus.cc:2454] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:35.225472 22915 leader_election.cc:302] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:35.225607 24015 raft_consensus.cc:2781] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:35.225656 24015 raft_consensus.cc:482] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:35.225695 24015 raft_consensus.cc:3037] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.226366 24015 raft_consensus.cc:504] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.226616 24015 leader_election.cc:288] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:35.226948 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b22f8e50a7d24dbdab665e47120602ce" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:35.227031 22847 raft_consensus.cc:3037] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.227015 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b22f8e50a7d24dbdab665e47120602ce" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:35.227129 22586 raft_consensus.cc:3037] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.227741 22847 raft_consensus.cc:2454] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:35.227813 22586 raft_consensus.cc:2454] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:35.227960 22915 leader_election.cc:302] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:35.228121 24015 raft_consensus.cc:2781] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:35.228260 24015 raft_consensus.cc:686] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:35.228376 24015 consensus_queue.cc:237] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.228528 24081 raft_consensus.cc:482] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:35.228602 24081 raft_consensus.cc:504] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.228984 24081 leader_election.cc:288] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:35.229126 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "73ad3c23d96b43b1894f767ad5db1e2f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:35.229142 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "73ad3c23d96b43b1894f767ad5db1e2f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:35.229234 22586 raft_consensus.cc:2454] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:35.229246 22715 raft_consensus.cc:2454] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:35.229496 22912 leader_election.cc:302] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:35.229609 24081 raft_consensus.cc:2781] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:35.229650 24081 raft_consensus.cc:482] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:35.229689 24081 raft_consensus.cc:3037] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.230046 22442 catalog_manager.cc:5484] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:35.230207 24081 raft_consensus.cc:504] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.230479 24081 leader_election.cc:288] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:35.230716 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "73ad3c23d96b43b1894f767ad5db1e2f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:35.230819 22715 raft_consensus.cc:3037] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.230866 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "73ad3c23d96b43b1894f767ad5db1e2f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:35.230952 22586 raft_consensus.cc:3037] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.231679 22715 raft_consensus.cc:2454] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:35.231729 22586 raft_consensus.cc:2454] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:35.231962 22915 leader_election.cc:302] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:35.232117 24081 raft_consensus.cc:2781] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:35.232195 24081 raft_consensus.cc:686] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:35.232318 24081 consensus_queue.cc:237] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:35.233986 22442 catalog_manager.cc:5484] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:35.337157 24084 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:35.337366 24084 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:35.340070 24084 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_8'
I20240502 07:58:35.415508 22715 raft_consensus.cc:1261] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.415931 24059 consensus_queue.cc:1035] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:35.416278 22586 raft_consensus.cc:1261] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.416724 24059 consensus_queue.cc:1035] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:35.419842 22717 raft_consensus.cc:1261] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.419838 22586 raft_consensus.cc:1261] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.421211 22584 raft_consensus.cc:1261] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.421186 24015 consensus_queue.cc:1035] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:35.421468 24015 consensus_queue.cc:1035] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:35.425510 24081 consensus_queue.cc:1035] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:35.428968 22847 raft_consensus.cc:1261] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.429940 24081 consensus_queue.cc:1035] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 114.825 ms
time per row: 0.0574125 ms
I20240502 07:58:35.492033 22443 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_9"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:35.496759 22828 tablet_service.cc:1467] Processing CreateTablet for tablet cceaa8887081401dbc9154d265004c7b (DEFAULT_TABLE table=rebalance_test_table_9 [id=bc838e66a1294569a905266b5619a7d7]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.496938 22696 tablet_service.cc:1467] Processing CreateTablet for tablet cceaa8887081401dbc9154d265004c7b (DEFAULT_TABLE table=rebalance_test_table_9 [id=bc838e66a1294569a905266b5619a7d7]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.497061 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cceaa8887081401dbc9154d265004c7b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.497234 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cceaa8887081401dbc9154d265004c7b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.497406 22959 tablet_service.cc:1467] Processing CreateTablet for tablet cceaa8887081401dbc9154d265004c7b (DEFAULT_TABLE table=rebalance_test_table_9 [id=bc838e66a1294569a905266b5619a7d7]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.497656 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cceaa8887081401dbc9154d265004c7b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.498718 23966 tablet_bootstrap.cc:492] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:35.499186 23964 tablet_bootstrap.cc:492] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:35.499573 23966 tablet_bootstrap.cc:654] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.499634 23963 tablet_bootstrap.cc:492] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:35.499951 23964 tablet_bootstrap.cc:654] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.500442 23963 tablet_bootstrap.cc:654] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.500949 22697 tablet_service.cc:1467] Processing CreateTablet for tablet ee06ff655f9c46c1bb06af2dd539fde7 (DEFAULT_TABLE table=rebalance_test_table_9 [id=bc838e66a1294569a905266b5619a7d7]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.501153 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ee06ff655f9c46c1bb06af2dd539fde7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.501129 22827 tablet_service.cc:1467] Processing CreateTablet for tablet ee06ff655f9c46c1bb06af2dd539fde7 (DEFAULT_TABLE table=rebalance_test_table_9 [id=bc838e66a1294569a905266b5619a7d7]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.501236 22959 tablet_service.cc:1467] Processing CreateTablet for tablet cfff02438656430686be0f03f5c0b8ab (DEFAULT_TABLE table=rebalance_test_table_9 [id=bc838e66a1294569a905266b5619a7d7]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.501327 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ee06ff655f9c46c1bb06af2dd539fde7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.501436 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cfff02438656430686be0f03f5c0b8ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.502954 22566 tablet_service.cc:1467] Processing CreateTablet for tablet ee06ff655f9c46c1bb06af2dd539fde7 (DEFAULT_TABLE table=rebalance_test_table_9 [id=bc838e66a1294569a905266b5619a7d7]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.503244 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ee06ff655f9c46c1bb06af2dd539fde7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.503468 23964 tablet_bootstrap.cc:492] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:35.503474 23966 tablet_bootstrap.cc:492] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:35.503548 23964 ts_tablet_manager.cc:1367] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:35.503548 23966 ts_tablet_manager.cc:1367] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:35.503933 23964 raft_consensus.cc:348] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.503933 23966 raft_consensus.cc:348] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.504032 23964 raft_consensus.cc:374] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.504032 23966 raft_consensus.cc:374] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.504065 23966 raft_consensus.cc:729] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.504065 23964 raft_consensus.cc:729] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.504171 23966 consensus_queue.cc:260] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.504171 23964 consensus_queue.cc:260] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.504415 23966 ts_tablet_manager.cc:1398] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:35.504415 23964 ts_tablet_manager.cc:1398] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:35.504549 23966 tablet_bootstrap.cc:492] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:35.504554 23964 tablet_bootstrap.cc:492] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:35.504979 23962 tablet_bootstrap.cc:492] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:35.505324 23966 tablet_bootstrap.cc:654] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.505612 22696 tablet_service.cc:1467] Processing CreateTablet for tablet cfff02438656430686be0f03f5c0b8ab (DEFAULT_TABLE table=rebalance_test_table_9 [id=bc838e66a1294569a905266b5619a7d7]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.505678 23962 tablet_bootstrap.cc:654] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.505817 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cfff02438656430686be0f03f5c0b8ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.506449 23964 tablet_bootstrap.cc:654] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.507575 23963 tablet_bootstrap.cc:492] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:35.507653 23963 ts_tablet_manager.cc:1367] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.000s
I20240502 07:58:35.507726 23964 tablet_bootstrap.cc:492] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:35.507792 23964 ts_tablet_manager.cc:1367] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:35.508047 23963 raft_consensus.cc:348] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.508143 23963 raft_consensus.cc:374] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.508167 23963 raft_consensus.cc:729] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.508211 23964 raft_consensus.cc:348] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.508268 23963 consensus_queue.cc:260] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.508304 23964 raft_consensus.cc:374] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.508337 23962 tablet_bootstrap.cc:492] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:35.508337 23964 raft_consensus.cc:729] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.508404 23962 ts_tablet_manager.cc:1367] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:35.508446 23964 consensus_queue.cc:260] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.508523 23963 ts_tablet_manager.cc:1398] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:35.508662 23963 tablet_bootstrap.cc:492] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:35.508750 23964 ts_tablet_manager.cc:1398] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:35.508908 23964 tablet_bootstrap.cc:492] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:35.509192 23962 raft_consensus.cc:348] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.509289 23962 raft_consensus.cc:374] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.509321 23962 raft_consensus.cc:729] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.509439 23966 tablet_bootstrap.cc:492] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:35.509434 23962 consensus_queue.cc:260] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.509510 23966 ts_tablet_manager.cc:1367] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:35.509550 23963 tablet_bootstrap.cc:654] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.509675 23962 ts_tablet_manager.cc:1398] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:35.509871 23964 tablet_bootstrap.cc:654] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.509943 23966 raft_consensus.cc:348] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.510028 23966 raft_consensus.cc:374] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.510057 23966 raft_consensus.cc:729] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.510037 22828 tablet_service.cc:1467] Processing CreateTablet for tablet cfff02438656430686be0f03f5c0b8ab (DEFAULT_TABLE table=rebalance_test_table_9 [id=bc838e66a1294569a905266b5619a7d7]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.510177 23966 consensus_queue.cc:260] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.510260 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cfff02438656430686be0f03f5c0b8ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.510387 23966 ts_tablet_manager.cc:1398] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:35.512239 23966 tablet_bootstrap.cc:492] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:35.512492 23964 tablet_bootstrap.cc:492] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:35.512576 23964 ts_tablet_manager.cc:1367] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:35.513120 23966 tablet_bootstrap.cc:654] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.513217 23963 tablet_bootstrap.cc:492] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:35.513285 23963 ts_tablet_manager.cc:1367] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:58:35.513350 23964 raft_consensus.cc:348] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.513437 23964 raft_consensus.cc:374] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.513469 23964 raft_consensus.cc:729] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.513573 23964 consensus_queue.cc:260] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.513703 23963 raft_consensus.cc:348] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.513782 23963 raft_consensus.cc:374] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.513811 23963 raft_consensus.cc:729] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.513821 23964 ts_tablet_manager.cc:1398] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:35.513906 23963 consensus_queue.cc:260] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.514181 23963 ts_tablet_manager.cc:1398] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:35.514420 23966 tablet_bootstrap.cc:492] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:35.514467 23966 ts_tablet_manager.cc:1367] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.002s
I20240502 07:58:35.514948 23966 raft_consensus.cc:348] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.515041 23966 raft_consensus.cc:374] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.515071 23966 raft_consensus.cc:729] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.515208 23966 consensus_queue.cc:260] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.515586 23966 ts_tablet_manager.cc:1398] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:35.524026 24029 raft_consensus.cc:482] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:35.524103 24029 raft_consensus.cc:504] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.524411 24029 leader_election.cc:288] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:35.524706 22585 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ee06ff655f9c46c1bb06af2dd539fde7" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:35.524749 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ee06ff655f9c46c1bb06af2dd539fde7" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:35.524811 22585 raft_consensus.cc:2454] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:35.524849 22715 raft_consensus.cc:2454] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:35.525054 22784 leader_election.cc:302] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:35.525233 24029 raft_consensus.cc:2781] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:35.525272 24029 raft_consensus.cc:482] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:35.525310 24029 raft_consensus.cc:3037] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.525986 24029 raft_consensus.cc:504] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.526278 24029 leader_election.cc:288] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:35.526502 22585 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ee06ff655f9c46c1bb06af2dd539fde7" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:35.526607 22585 raft_consensus.cc:3037] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.526634 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ee06ff655f9c46c1bb06af2dd539fde7" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:35.526731 22715 raft_consensus.cc:3037] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.527293 22585 raft_consensus.cc:2454] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:35.527330 22715 raft_consensus.cc:2454] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:35.527575 22784 leader_election.cc:302] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:35.527776 24029 raft_consensus.cc:2781] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:35.527848 24029 raft_consensus.cc:686] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:35.527957 24029 consensus_queue.cc:237] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:35.529317 22443 catalog_manager.cc:5484] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:35.594223 24114 raft_consensus.cc:482] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:35.594326 24114 raft_consensus.cc:504] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.594671 24114 leader_election.cc:288] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:35.594986 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cfff02438656430686be0f03f5c0b8ab" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:35.594974 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cfff02438656430686be0f03f5c0b8ab" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:35.595101 22715 raft_consensus.cc:2454] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:35.595101 22847 raft_consensus.cc:2454] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:35.595360 22912 leader_election.cc:302] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:35.595535 24114 raft_consensus.cc:2781] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:35.595626 24114 raft_consensus.cc:482] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:35.595662 24114 raft_consensus.cc:3037] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.596688 24114 raft_consensus.cc:504] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.596995 24114 leader_election.cc:288] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:35.597275 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cfff02438656430686be0f03f5c0b8ab" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:35.597288 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cfff02438656430686be0f03f5c0b8ab" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:35.597404 22847 raft_consensus.cc:3037] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.597402 22715 raft_consensus.cc:3037] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.598235 22847 raft_consensus.cc:2454] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:35.598235 22715 raft_consensus.cc:2454] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:35.598472 22912 leader_election.cc:302] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:35.598696 24114 raft_consensus.cc:2781] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:35.598781 24114 raft_consensus.cc:686] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:35.598938 24114 consensus_queue.cc:237] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.600486 22443 catalog_manager.cc:5484] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:35.611554 23908 raft_consensus.cc:482] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:35.611667 23908 raft_consensus.cc:504] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.612040 23908 leader_election.cc:288] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:35.612366 22977 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cceaa8887081401dbc9154d265004c7b" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:35.612381 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cceaa8887081401dbc9154d265004c7b" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:35.612468 22977 raft_consensus.cc:2454] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:35.612498 22847 raft_consensus.cc:2454] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:35.612743 22651 leader_election.cc:302] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:35.612919 23908 raft_consensus.cc:2781] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:35.612983 23908 raft_consensus.cc:482] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:35.613024 23908 raft_consensus.cc:3037] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.613866 23908 raft_consensus.cc:504] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.614156 23908 leader_election.cc:288] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:35.614403 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cceaa8887081401dbc9154d265004c7b" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:35.614408 22977 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cceaa8887081401dbc9154d265004c7b" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:35.614490 22977 raft_consensus.cc:3037] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.614495 22847 raft_consensus.cc:3037] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.615160 22977 raft_consensus.cc:2454] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:35.615172 22847 raft_consensus.cc:2454] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:35.615409 22651 leader_election.cc:302] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:35.615572 23908 raft_consensus.cc:2781] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:35.615752 23908 raft_consensus.cc:686] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:35.615890 23908 consensus_queue.cc:237] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.617468 22443 catalog_manager.cc:5484] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:35.724486 24134 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:35.724742 24134 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:35.727499 24134 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_9'
I20240502 07:58:35.809574 22715 raft_consensus.cc:1261] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.809687 22847 raft_consensus.cc:1261] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.810099 24081 consensus_queue.cc:1035] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:35.810351 24081 consensus_queue.cc:1035] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:35.816094 22715 raft_consensus.cc:1261] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.817220 22585 raft_consensus.cc:1261] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.817741 24069 consensus_queue.cc:1035] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:35.818010 24069 consensus_queue.cc:1035] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:35.820830 22977 raft_consensus.cc:1261] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.821880 23908 consensus_queue.cc:1035] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:35.833853 23861 mvcc.cc:205] Tried to move back new op lower bound from 7023151987963379712 to 7023151986803625984. Current Snapshot: MvccSnapshot[applied={T|T < 7023151987963379712}]
I20240502 07:58:35.835232 22847 raft_consensus.cc:1261] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:35.837203 23908 consensus_queue.cc:1035] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:35.843624 23862 mvcc.cc:205] Tried to move back new op lower bound from 7023151987996962816 to 7023151987163795456. Current Snapshot: MvccSnapshot[applied={T|T < 7023151987996962816 or (T in {7023151987996962816})}]
I20240502 07:58:35.845265 23952 mvcc.cc:205] Tried to move back new op lower bound from 7023151987996962816 to 7023151987163795456. Current Snapshot: MvccSnapshot[applied={T|T < 7023151987996962816 or (T in {7023151987996962816})}]
INSERT report
rows total: 2000
time total: 107.534 ms
time per row: 0.0537669 ms
I20240502 07:58:35.874788 22438 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_10"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:35.879606 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 398405a3694c4d29b0627d2d74ebadc6 (DEFAULT_TABLE table=rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.879689 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 398405a3694c4d29b0627d2d74ebadc6 (DEFAULT_TABLE table=rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.879889 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 398405a3694c4d29b0627d2d74ebadc6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.879905 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 398405a3694c4d29b0627d2d74ebadc6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.880136 22958 tablet_service.cc:1467] Processing CreateTablet for tablet 1e370c8fe68b4392ba3720533e538de3 (DEFAULT_TABLE table=rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.880398 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1e370c8fe68b4392ba3720533e538de3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.881132 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 0e01743578f947cca1db2baec4fdee12 (DEFAULT_TABLE table=rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.881392 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0e01743578f947cca1db2baec4fdee12. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.881846 23963 tablet_bootstrap.cc:492] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:35.882018 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 1e370c8fe68b4392ba3720533e538de3 (DEFAULT_TABLE table=rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.882263 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1e370c8fe68b4392ba3720533e538de3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.882731 23963 tablet_bootstrap.cc:654] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.883114 23964 tablet_bootstrap.cc:492] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:35.883291 22696 tablet_service.cc:1467] Processing CreateTablet for tablet 398405a3694c4d29b0627d2d74ebadc6 (DEFAULT_TABLE table=rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.883466 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 398405a3694c4d29b0627d2d74ebadc6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.883889 23964 tablet_bootstrap.cc:654] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.884109 23966 tablet_bootstrap.cc:492] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:35.884171 23962 tablet_bootstrap.cc:492] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:35.884899 23964 tablet_bootstrap.cc:492] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:35.884912 23966 tablet_bootstrap.cc:654] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.884966 23964 ts_tablet_manager.cc:1367] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:35.885355 23964 raft_consensus.cc:348] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.885453 23964 raft_consensus.cc:374] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.885479 23964 raft_consensus.cc:729] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.885573 23964 consensus_queue.cc:260] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.885800 23964 ts_tablet_manager.cc:1398] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:35.885926 23964 tablet_bootstrap.cc:492] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:35.885916 23966 tablet_bootstrap.cc:492] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:35.885982 23966 ts_tablet_manager.cc:1367] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:35.886090 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 0e01743578f947cca1db2baec4fdee12 (DEFAULT_TABLE table=rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.886274 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0e01743578f947cca1db2baec4fdee12. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.886371 23966 raft_consensus.cc:348] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.886452 23966 raft_consensus.cc:374] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.886487 23966 raft_consensus.cc:729] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.886608 23966 consensus_queue.cc:260] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.886772 23964 tablet_bootstrap.cc:654] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.886826 23966 ts_tablet_manager.cc:1398] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:35.888175 23962 tablet_bootstrap.cc:654] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.888857 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 1e370c8fe68b4392ba3720533e538de3 (DEFAULT_TABLE table=rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.889024 23962 tablet_bootstrap.cc:492] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:35.889045 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1e370c8fe68b4392ba3720533e538de3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.889084 23962 ts_tablet_manager.cc:1367] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:35.889411 23962 raft_consensus.cc:348] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:35.889492 23962 raft_consensus.cc:374] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.889519 23962 raft_consensus.cc:729] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.889616 23963 tablet_bootstrap.cc:492] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:35.889649 23962 consensus_queue.cc:260] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:35.889688 23963 ts_tablet_manager.cc:1367] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.000s
I20240502 07:58:35.889919 23962 ts_tablet_manager.cc:1398] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:35.890064 23962 tablet_bootstrap.cc:492] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:35.890061 23963 raft_consensus.cc:348] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:35.890149 23963 raft_consensus.cc:374] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.890185 23963 raft_consensus.cc:729] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.890304 23963 consensus_queue.cc:260] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:35.890620 23963 ts_tablet_manager.cc:1398] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:35.890801 23963 tablet_bootstrap.cc:492] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:35.890882 23962 tablet_bootstrap.cc:654] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.891566 23963 tablet_bootstrap.cc:654] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.891775 22958 tablet_service.cc:1467] Processing CreateTablet for tablet 0e01743578f947cca1db2baec4fdee12 (DEFAULT_TABLE table=rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:35.892017 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0e01743578f947cca1db2baec4fdee12. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:35.892638 23962 tablet_bootstrap.cc:492] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:35.892751 23962 ts_tablet_manager.cc:1367] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:35.893165 23962 raft_consensus.cc:348] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.893268 23962 raft_consensus.cc:374] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.893307 23962 raft_consensus.cc:729] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.893410 23962 consensus_queue.cc:260] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.893668 23962 ts_tablet_manager.cc:1398] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:35.893790 23962 tablet_bootstrap.cc:492] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:35.894552 23963 tablet_bootstrap.cc:492] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:35.894575 23962 tablet_bootstrap.cc:654] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.894609 23963 ts_tablet_manager.cc:1367] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:35.894937 23963 raft_consensus.cc:348] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.895010 23963 raft_consensus.cc:374] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.895035 23963 raft_consensus.cc:729] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.895128 23963 consensus_queue.cc:260] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.895354 23963 ts_tablet_manager.cc:1398] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:35.895488 23963 tablet_bootstrap.cc:492] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:35.895747 23962 tablet_bootstrap.cc:492] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:35.895823 23962 ts_tablet_manager.cc:1367] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:35.896126 23962 raft_consensus.cc:348] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.896214 23963 tablet_bootstrap.cc:654] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:35.896211 23962 raft_consensus.cc:374] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.896247 23962 raft_consensus.cc:729] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.896385 23962 consensus_queue.cc:260] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.896693 23962 ts_tablet_manager.cc:1398] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:35.897327 23963 tablet_bootstrap.cc:492] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:35.897394 23963 ts_tablet_manager.cc:1367] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:35.897722 23963 raft_consensus.cc:348] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.897825 23963 raft_consensus.cc:374] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.897852 23963 raft_consensus.cc:729] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.897979 23963 consensus_queue.cc:260] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.898248 23963 ts_tablet_manager.cc:1398] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:35.901273 23964 tablet_bootstrap.cc:492] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:35.901353 23964 ts_tablet_manager.cc:1367] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.015s user 0.001s sys 0.000s
I20240502 07:58:35.901706 23964 raft_consensus.cc:348] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:35.901798 23964 raft_consensus.cc:374] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:35.901823 23964 raft_consensus.cc:729] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:35.901921 23964 consensus_queue.cc:260] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:35.902199 23964 ts_tablet_manager.cc:1398] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:35.939720 24015 raft_consensus.cc:482] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:35.939829 24015 raft_consensus.cc:504] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.940129 24015 leader_election.cc:288] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:35.940464 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1e370c8fe68b4392ba3720533e538de3" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:35.940501 22585 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1e370c8fe68b4392ba3720533e538de3" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:35.940632 22847 raft_consensus.cc:2454] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:35.940632 22585 raft_consensus.cc:2454] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:35.940915 22915 leader_election.cc:302] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:35.941079 24015 raft_consensus.cc:2781] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:35.941133 24015 raft_consensus.cc:482] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:35.941177 24015 raft_consensus.cc:3037] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.941943 24015 raft_consensus.cc:504] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.942238 24015 leader_election.cc:288] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:35.942521 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1e370c8fe68b4392ba3720533e538de3" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:35.942618 22847 raft_consensus.cc:3037] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.942615 22585 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1e370c8fe68b4392ba3720533e538de3" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:35.942765 22585 raft_consensus.cc:3037] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.943347 22847 raft_consensus.cc:2454] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:35.943535 22585 raft_consensus.cc:2454] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:35.943593 22915 leader_election.cc:302] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:35.943729 24015 raft_consensus.cc:2781] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:35.943815 24015 raft_consensus.cc:686] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:35.943933 24015 consensus_queue.cc:237] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.945574 22438 catalog_manager.cc:5484] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:35.962163 23908 raft_consensus.cc:482] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:35.962258 23908 raft_consensus.cc:504] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.962616 23908 leader_election.cc:288] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:35.962926 22585 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0e01743578f947cca1db2baec4fdee12" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:35.962963 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0e01743578f947cca1db2baec4fdee12" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:35.963065 22979 raft_consensus.cc:2454] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:35.963060 22585 raft_consensus.cc:2454] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:35.963321 22651 leader_election.cc:302] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:35.963466 23908 raft_consensus.cc:2781] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:35.963522 23908 raft_consensus.cc:482] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:35.963578 23908 raft_consensus.cc:3037] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.964290 23908 raft_consensus.cc:504] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.964648 23908 leader_election.cc:288] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:35.964943 22585 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0e01743578f947cca1db2baec4fdee12" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:35.964982 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0e01743578f947cca1db2baec4fdee12" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:35.965030 22585 raft_consensus.cc:3037] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.965095 22979 raft_consensus.cc:3037] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:35.965708 22585 raft_consensus.cc:2454] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:35.965762 22979 raft_consensus.cc:2454] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:35.965947 22653 leader_election.cc:302] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:35.966164 23908 raft_consensus.cc:2781] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:35.966230 23908 raft_consensus.cc:686] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:35.966357 23908 consensus_queue.cc:237] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:35.968011 22438 catalog_manager.cc:5484] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:36.008620 24015 raft_consensus.cc:482] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:36.008759 24015 raft_consensus.cc:504] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.009625 24015 leader_election.cc:288] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:36.009790 22585 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "398405a3694c4d29b0627d2d74ebadc6" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:36.009959 22585 raft_consensus.cc:2454] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:36.009948 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "398405a3694c4d29b0627d2d74ebadc6" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:36.010090 22715 raft_consensus.cc:2454] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:36.010200 22915 leader_election.cc:302] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:36.010382 24015 raft_consensus.cc:2781] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:36.010466 24015 raft_consensus.cc:482] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:36.010562 24015 raft_consensus.cc:3037] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.011353 24015 raft_consensus.cc:504] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.011682 24015 leader_election.cc:288] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:36.012003 22585 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "398405a3694c4d29b0627d2d74ebadc6" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:36.012115 22585 raft_consensus.cc:3037] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.012118 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "398405a3694c4d29b0627d2d74ebadc6" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:36.012223 22715 raft_consensus.cc:3037] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.012975 22585 raft_consensus.cc:2454] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:36.013044 22715 raft_consensus.cc:2454] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:36.013268 22915 leader_election.cc:302] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:36.013502 24015 raft_consensus.cc:2781] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:36.013664 24015 raft_consensus.cc:686] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:36.013813 24015 consensus_queue.cc:237] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.015440 22438 catalog_manager.cc:5484] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:36.141523 24187 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:36.141770 24187 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:36.144814 24187 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_10'
I20240502 07:58:36.218845 22979 raft_consensus.cc:1261] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:36.219245 23908 consensus_queue.cc:1035] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:36.219959 22715 raft_consensus.cc:1261] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:36.220227 22585 raft_consensus.cc:1261] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:36.220432 24015 consensus_queue.cc:1035] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:36.220793 24015 consensus_queue.cc:1035] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:36.225728 22584 raft_consensus.cc:1261] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:36.226068 23908 consensus_queue.cc:1035] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:36.226192 22847 raft_consensus.cc:1261] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:36.226773 22586 raft_consensus.cc:1261] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:36.227084 24114 consensus_queue.cc:1035] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:36.227173 24015 consensus_queue.cc:1035] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:36.244820 24116 mvcc.cc:205] Tried to move back new op lower bound from 7023151989627666432 to 7023151988599570432. Current Snapshot: MvccSnapshot[applied={T|T < 7023151989627666432}]
INSERT report
rows total: 2000
time total: 95.907 ms
time per row: 0.0479535 ms
I20240502 07:58:36.280223 22438 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_11"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:36.284996 22828 tablet_service.cc:1467] Processing CreateTablet for tablet e25ba5cc77334a3595a4575fa98476f0 (DEFAULT_TABLE table=rebalance_test_table_11 [id=0c1156cac15d48028ebf5c07c67ea832]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.284996 22566 tablet_service.cc:1467] Processing CreateTablet for tablet e25ba5cc77334a3595a4575fa98476f0 (DEFAULT_TABLE table=rebalance_test_table_11 [id=0c1156cac15d48028ebf5c07c67ea832]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.285264 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e25ba5cc77334a3595a4575fa98476f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.285264 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e25ba5cc77334a3595a4575fa98476f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.286306 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 9d2283e5d28347519b38e876d840608e (DEFAULT_TABLE table=rebalance_test_table_11 [id=0c1156cac15d48028ebf5c07c67ea832]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.286597 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9d2283e5d28347519b38e876d840608e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.286756 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 28d5d20938a4472caf029c8d1ac89b82 (DEFAULT_TABLE table=rebalance_test_table_11 [id=0c1156cac15d48028ebf5c07c67ea832]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.286958 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 28d5d20938a4472caf029c8d1ac89b82. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.287209 23962 tablet_bootstrap.cc:492] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:36.287209 23966 tablet_bootstrap.cc:492] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:36.288005 23966 tablet_bootstrap.cc:654] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.288005 23962 tablet_bootstrap.cc:654] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.288775 23964 tablet_bootstrap.cc:492] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:36.288923 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 9d2283e5d28347519b38e876d840608e (DEFAULT_TABLE table=rebalance_test_table_11 [id=0c1156cac15d48028ebf5c07c67ea832]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.289022 23962 tablet_bootstrap.cc:492] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:36.289096 23962 ts_tablet_manager.cc:1367] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:36.289137 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9d2283e5d28347519b38e876d840608e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.289538 23964 tablet_bootstrap.cc:654] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.289497 23962 raft_consensus.cc:348] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.289580 23962 raft_consensus.cc:374] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.289608 23962 raft_consensus.cc:729] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.289700 23962 consensus_queue.cc:260] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.289959 23962 ts_tablet_manager.cc:1398] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:36.286306 22696 tablet_service.cc:1467] Processing CreateTablet for tablet e25ba5cc77334a3595a4575fa98476f0 (DEFAULT_TABLE table=rebalance_test_table_11 [id=0c1156cac15d48028ebf5c07c67ea832]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.290128 23962 tablet_bootstrap.cc:492] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:36.290185 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e25ba5cc77334a3595a4575fa98476f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.290931 23962 tablet_bootstrap.cc:654] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.291538 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 28d5d20938a4472caf029c8d1ac89b82 (DEFAULT_TABLE table=rebalance_test_table_11 [id=0c1156cac15d48028ebf5c07c67ea832]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.291749 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 28d5d20938a4472caf029c8d1ac89b82. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.292150 23962 tablet_bootstrap.cc:492] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:36.292227 23962 ts_tablet_manager.cc:1367] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:36.292635 23962 raft_consensus.cc:348] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.292778 23962 raft_consensus.cc:374] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.292814 23962 raft_consensus.cc:729] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.292948 23962 consensus_queue.cc:260] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.293329 23962 ts_tablet_manager.cc:1398] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:36.294047 23966 tablet_bootstrap.cc:492] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:36.294118 23966 ts_tablet_manager.cc:1367] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 07:58:36.294241 22958 tablet_service.cc:1467] Processing CreateTablet for tablet 28d5d20938a4472caf029c8d1ac89b82 (DEFAULT_TABLE table=rebalance_test_table_11 [id=0c1156cac15d48028ebf5c07c67ea832]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.294433 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 9d2283e5d28347519b38e876d840608e (DEFAULT_TABLE table=rebalance_test_table_11 [id=0c1156cac15d48028ebf5c07c67ea832]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.294497 23966 raft_consensus.cc:348] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.294611 23966 raft_consensus.cc:374] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.294512 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 28d5d20938a4472caf029c8d1ac89b82. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.294656 23966 raft_consensus.cc:729] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.294834 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9d2283e5d28347519b38e876d840608e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.294787 23966 consensus_queue.cc:260] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.295038 23966 ts_tablet_manager.cc:1398] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:36.295214 23966 tablet_bootstrap.cc:492] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:36.295253 23964 tablet_bootstrap.cc:492] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:36.295328 23964 ts_tablet_manager.cc:1367] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 07:58:36.295750 23964 raft_consensus.cc:348] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.295850 23964 raft_consensus.cc:374] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.295877 23964 raft_consensus.cc:729] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.295984 23966 tablet_bootstrap.cc:654] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.295984 23964 consensus_queue.cc:260] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.296200 23964 ts_tablet_manager.cc:1398] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:36.296343 23964 tablet_bootstrap.cc:492] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:36.296859 23963 tablet_bootstrap.cc:492] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:36.296898 23966 tablet_bootstrap.cc:492] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:36.296957 23966 ts_tablet_manager.cc:1367] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.001s
I20240502 07:58:36.297093 23964 tablet_bootstrap.cc:654] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.297291 23966 raft_consensus.cc:348] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.297374 23966 raft_consensus.cc:374] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.297402 23966 raft_consensus.cc:729] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.297516 23966 consensus_queue.cc:260] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.297830 23966 ts_tablet_manager.cc:1398] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:36.297932 23966 tablet_bootstrap.cc:492] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:36.297930 23963 tablet_bootstrap.cc:654] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.298508 23966 tablet_bootstrap.cc:654] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.299023 23963 tablet_bootstrap.cc:492] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:36.299086 23963 ts_tablet_manager.cc:1367] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:36.299475 23963 raft_consensus.cc:348] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.299568 23963 raft_consensus.cc:374] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.299598 23963 raft_consensus.cc:729] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.299705 23963 consensus_queue.cc:260] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.299940 23963 ts_tablet_manager.cc:1398] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:36.300987 23963 tablet_bootstrap.cc:492] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:36.301612 23964 tablet_bootstrap.cc:492] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:36.301673 23964 ts_tablet_manager.cc:1367] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:36.302191 23964 raft_consensus.cc:348] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.302443 23964 raft_consensus.cc:374] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.302492 23964 raft_consensus.cc:729] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.302616 23964 consensus_queue.cc:260] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.302947 23964 ts_tablet_manager.cc:1398] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:36.303467 23963 tablet_bootstrap.cc:654] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.304220 23966 tablet_bootstrap.cc:492] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:36.304293 23966 ts_tablet_manager.cc:1367] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.001s
I20240502 07:58:36.304466 23963 tablet_bootstrap.cc:492] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:36.304553 23963 ts_tablet_manager.cc:1367] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:36.304687 23966 raft_consensus.cc:348] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.304775 23966 raft_consensus.cc:374] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.304811 23966 raft_consensus.cc:729] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.304912 23966 consensus_queue.cc:260] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.304991 23963 raft_consensus.cc:348] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.305079 23963 raft_consensus.cc:374] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.305105 23963 raft_consensus.cc:729] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.305136 23966 ts_tablet_manager.cc:1398] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:36.305213 23963 consensus_queue.cc:260] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.305611 23963 ts_tablet_manager.cc:1398] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:36.382694 24220 raft_consensus.cc:482] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:36.382802 24220 raft_consensus.cc:504] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.383199 24220 leader_election.cc:288] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:36.383558 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e25ba5cc77334a3595a4575fa98476f0" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:36.383626 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e25ba5cc77334a3595a4575fa98476f0" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:36.383697 22848 raft_consensus.cc:2454] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:36.383777 22584 raft_consensus.cc:2454] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:36.383911 22653 leader_election.cc:302] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:36.384052 24220 raft_consensus.cc:2781] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:36.384106 24220 raft_consensus.cc:482] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:36.384150 24220 raft_consensus.cc:3037] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.384907 24220 raft_consensus.cc:504] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.385159 24220 leader_election.cc:288] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:36.385478 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e25ba5cc77334a3595a4575fa98476f0" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:36.385547 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e25ba5cc77334a3595a4575fa98476f0" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:36.385623 22848 raft_consensus.cc:3037] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.385648 22584 raft_consensus.cc:3037] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.386498 22584 raft_consensus.cc:2454] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:36.386498 22848 raft_consensus.cc:2454] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:36.386768 22653 leader_election.cc:302] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:36.386911 24220 raft_consensus.cc:2781] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:36.386989 24220 raft_consensus.cc:686] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:36.387118 24220 consensus_queue.cc:237] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.388633 22438 catalog_manager.cc:5484] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:36.494750 24015 raft_consensus.cc:482] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:36.494846 24015 raft_consensus.cc:504] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.495159 24015 leader_election.cc:288] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:36.495482 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28d5d20938a4472caf029c8d1ac89b82" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:36.495536 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28d5d20938a4472caf029c8d1ac89b82" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:36.495606 22848 raft_consensus.cc:2454] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:36.495649 22584 raft_consensus.cc:2454] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:36.495832 22915 leader_election.cc:302] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:36.495977 24015 raft_consensus.cc:2781] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:36.496030 24015 raft_consensus.cc:482] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:36.496073 24015 raft_consensus.cc:3037] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.496879 24015 raft_consensus.cc:504] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.497131 24015 leader_election.cc:288] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:36.497390 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28d5d20938a4472caf029c8d1ac89b82" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:36.497437 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28d5d20938a4472caf029c8d1ac89b82" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:36.497485 22848 raft_consensus.cc:3037] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.497547 22584 raft_consensus.cc:3037] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.498399 22848 raft_consensus.cc:2454] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:36.498448 22584 raft_consensus.cc:2454] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:36.498685 22915 leader_election.cc:302] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:36.498807 24015 raft_consensus.cc:2781] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:36.498885 24015 raft_consensus.cc:686] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:36.499006 24015 consensus_queue.cc:237] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.500619 22438 catalog_manager.cc:5484] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:36.639966 24114 raft_consensus.cc:482] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:36.640105 24114 raft_consensus.cc:504] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.640504 24114 leader_election.cc:288] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:36.640866 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d2283e5d28347519b38e876d840608e" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:36.640869 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d2283e5d28347519b38e876d840608e" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:36.640980 22715 raft_consensus.cc:2454] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:36.640991 22848 raft_consensus.cc:2454] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:36.641230 22915 leader_election.cc:302] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:36.641386 24114 raft_consensus.cc:2781] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:36.641454 24114 raft_consensus.cc:482] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:36.641494 24114 raft_consensus.cc:3037] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.642388 24114 raft_consensus.cc:504] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.642673 24114 leader_election.cc:288] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:36.642920 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d2283e5d28347519b38e876d840608e" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:36.643020 22715 raft_consensus.cc:3037] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.643069 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d2283e5d28347519b38e876d840608e" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:36.643182 22848 raft_consensus.cc:3037] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.643752 22715 raft_consensus.cc:2454] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:36.643872 22848 raft_consensus.cc:2454] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:36.644001 22912 leader_election.cc:302] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:36.644162 24114 raft_consensus.cc:2781] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:36.644238 24114 raft_consensus.cc:686] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:36.644372 24114 consensus_queue.cc:237] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.646039 22438 catalog_manager.cc:5484] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:36.763255 24235 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:36.763499 24235 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:36.766264 24235 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_11'
I20240502 07:58:36.855505 22584 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:36.855820 22848 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:36.856314 24132 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:36.856582 24132 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:36.862432 22715 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:36.862941 24081 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:36.863353 22846 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:36.864102 22846 raft_consensus.cc:1261] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:36.864127 22586 raft_consensus.cc:1261] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:36.864463 24081 consensus_queue.cc:1035] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:36.864637 24015 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:36.867981 24015 consensus_queue.cc:1035] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 120.362 ms
time per row: 0.0601808 ms
I20240502 07:58:36.933171 22435 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_12"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:36.938073 22696 tablet_service.cc:1467] Processing CreateTablet for tablet 90480cc47cac4c84994bfcb9bc8a2dbf (DEFAULT_TABLE table=rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.938092 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 90480cc47cac4c84994bfcb9bc8a2dbf (DEFAULT_TABLE table=rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.938351 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 90480cc47cac4c84994bfcb9bc8a2dbf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.938349 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 90480cc47cac4c84994bfcb9bc8a2dbf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.938661 22827 tablet_service.cc:1467] Processing CreateTablet for tablet eecae5d013f543bf8eed1f2488757115 (DEFAULT_TABLE table=rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.938843 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eecae5d013f543bf8eed1f2488757115. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.940301 24298 tablet_bootstrap.cc:492] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:36.940526 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 90480cc47cac4c84994bfcb9bc8a2dbf (DEFAULT_TABLE table=rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.940641 22565 tablet_service.cc:1467] Processing CreateTablet for tablet eecae5d013f543bf8eed1f2488757115 (DEFAULT_TABLE table=rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.940795 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 90480cc47cac4c84994bfcb9bc8a2dbf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.940888 22565 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eecae5d013f543bf8eed1f2488757115. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.940848 22958 tablet_service.cc:1467] Processing CreateTablet for tablet f1830cb65a294734ad80f274b89ac185 (DEFAULT_TABLE table=rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.941067 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f1830cb65a294734ad80f274b89ac185. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.941112 24298 tablet_bootstrap.cc:654] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.942597 24301 tablet_bootstrap.cc:492] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:36.942946 22566 tablet_service.cc:1467] Processing CreateTablet for tablet f1830cb65a294734ad80f274b89ac185 (DEFAULT_TABLE table=rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.943147 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f1830cb65a294734ad80f274b89ac185. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.943243 22959 tablet_service.cc:1467] Processing CreateTablet for tablet eecae5d013f543bf8eed1f2488757115 (DEFAULT_TABLE table=rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.943457 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eecae5d013f543bf8eed1f2488757115. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.943598 24298 tablet_bootstrap.cc:492] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:36.943647 24298 ts_tablet_manager.cc:1367] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:36.943907 24298 raft_consensus.cc:348] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.943970 24298 raft_consensus.cc:374] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.943987 24298 raft_consensus.cc:729] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.944077 24298 consensus_queue.cc:260] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.944332 24298 ts_tablet_manager.cc:1398] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:36.944507 24298 tablet_bootstrap.cc:492] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:36.945228 24299 tablet_bootstrap.cc:492] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:36.945272 24298 tablet_bootstrap.cc:654] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.945892 22696 tablet_service.cc:1467] Processing CreateTablet for tablet f1830cb65a294734ad80f274b89ac185 (DEFAULT_TABLE table=rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:36.946043 24299 tablet_bootstrap.cc:654] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.946077 24301 tablet_bootstrap.cc:654] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.946105 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f1830cb65a294734ad80f274b89ac185. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:36.947034 24298 tablet_bootstrap.cc:492] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:36.947101 24298 ts_tablet_manager.cc:1367] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:36.947149 24301 tablet_bootstrap.cc:492] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:36.947233 24301 ts_tablet_manager.cc:1367] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:36.947496 24298 raft_consensus.cc:348] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.947600 24298 raft_consensus.cc:374] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.947633 24298 raft_consensus.cc:729] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.947628 24301 raft_consensus.cc:348] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.947748 24298 consensus_queue.cc:260] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.947774 24301 raft_consensus.cc:374] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.947819 24301 raft_consensus.cc:729] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.947952 24301 consensus_queue.cc:260] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.948004 24298 ts_tablet_manager.cc:1398] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:36.948719 24299 tablet_bootstrap.cc:492] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:36.948801 24299 ts_tablet_manager.cc:1367] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:36.948997 24301 ts_tablet_manager.cc:1398] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:36.949141 24301 tablet_bootstrap.cc:492] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:36.949184 24299 raft_consensus.cc:348] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.949280 24299 raft_consensus.cc:374] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.949309 24299 raft_consensus.cc:729] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.949426 24299 consensus_queue.cc:260] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.950213 24299 ts_tablet_manager.cc:1398] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:36.950369 24299 tablet_bootstrap.cc:492] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:36.950510 24301 tablet_bootstrap.cc:654] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.951172 24299 tablet_bootstrap.cc:654] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.951773 24302 tablet_bootstrap.cc:492] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:36.952209 24299 tablet_bootstrap.cc:492] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:36.952286 24299 ts_tablet_manager.cc:1367] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:36.952620 24299 raft_consensus.cc:348] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.952790 24299 raft_consensus.cc:374] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.952840 24299 raft_consensus.cc:729] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.952987 24299 consensus_queue.cc:260] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.953246 24299 ts_tablet_manager.cc:1398] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:36.953275 24302 tablet_bootstrap.cc:654] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.953389 24301 tablet_bootstrap.cc:492] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:36.953455 24301 ts_tablet_manager.cc:1367] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:36.953869 24301 raft_consensus.cc:348] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.953974 24301 raft_consensus.cc:374] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.954010 24301 raft_consensus.cc:729] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.954114 24301 consensus_queue.cc:260] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.954362 24301 ts_tablet_manager.cc:1398] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:36.954502 24301 tablet_bootstrap.cc:492] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:36.955076 24302 tablet_bootstrap.cc:492] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:36.955152 24302 ts_tablet_manager.cc:1367] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
I20240502 07:58:36.955304 24301 tablet_bootstrap.cc:654] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.955567 24302 raft_consensus.cc:348] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.955659 24302 raft_consensus.cc:374] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.955696 24302 raft_consensus.cc:729] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.955807 24302 consensus_queue.cc:260] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.956072 24302 ts_tablet_manager.cc:1398] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:36.956193 24302 tablet_bootstrap.cc:492] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:36.956810 24302 tablet_bootstrap.cc:654] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:36.958475 24301 tablet_bootstrap.cc:492] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:36.958541 24301 ts_tablet_manager.cc:1367] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:36.958729 24302 tablet_bootstrap.cc:492] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:36.958822 24302 ts_tablet_manager.cc:1367] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:36.958849 24301 raft_consensus.cc:348] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.958935 24301 raft_consensus.cc:374] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.958968 24301 raft_consensus.cc:729] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.959070 24301 consensus_queue.cc:260] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.959211 24302 raft_consensus.cc:348] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.959282 24301 ts_tablet_manager.cc:1398] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:36.959291 24302 raft_consensus.cc:374] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:36.959329 24302 raft_consensus.cc:729] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:36.959465 24302 consensus_queue.cc:260] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.959803 24302 ts_tablet_manager.cc:1398] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:36.967291 24184 raft_consensus.cc:482] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:36.967357 24184 raft_consensus.cc:504] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.967633 24184 leader_election.cc:288] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:36.967890 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f1830cb65a294734ad80f274b89ac185" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:36.967972 22979 raft_consensus.cc:2454] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:36.968149 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f1830cb65a294734ad80f274b89ac185" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:36.968214 22520 leader_election.cc:302] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:36.968250 22715 raft_consensus.cc:2454] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:36.968355 24184 raft_consensus.cc:2781] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:36.968418 24184 raft_consensus.cc:482] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:36.968462 24184 raft_consensus.cc:3037] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.969148 24184 raft_consensus.cc:504] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.969410 24184 leader_election.cc:288] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:36.969671 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f1830cb65a294734ad80f274b89ac185" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:36.969668 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f1830cb65a294734ad80f274b89ac185" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:36.969746 22979 raft_consensus.cc:3037] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.969754 22715 raft_consensus.cc:3037] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.970388 22715 raft_consensus.cc:2454] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:36.970427 22979 raft_consensus.cc:2454] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:36.970675 22519 leader_election.cc:302] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:36.970846 24184 raft_consensus.cc:2781] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:36.970978 24184 raft_consensus.cc:686] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:36.971098 24184 consensus_queue.cc:237] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:36.972719 22441 catalog_manager.cc:5484] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:36.992841 24279 raft_consensus.cc:482] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:36.992928 24279 raft_consensus.cc:504] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.993237 24279 leader_election.cc:288] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:36.993593 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "90480cc47cac4c84994bfcb9bc8a2dbf" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:36.993700 22715 raft_consensus.cc:2454] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:36.993813 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "90480cc47cac4c84994bfcb9bc8a2dbf" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:36.993927 22584 raft_consensus.cc:2454] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:36.993921 22781 leader_election.cc:302] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:36.994057 24279 raft_consensus.cc:2781] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:36.994122 24279 raft_consensus.cc:482] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:36.994159 24279 raft_consensus.cc:3037] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.994817 24279 raft_consensus.cc:504] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.995038 24279 leader_election.cc:288] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:36.995317 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "90480cc47cac4c84994bfcb9bc8a2dbf" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:36.995368 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "90480cc47cac4c84994bfcb9bc8a2dbf" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:36.995407 22715 raft_consensus.cc:3037] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.995465 22584 raft_consensus.cc:3037] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:36.996088 22715 raft_consensus.cc:2454] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:36.996166 22584 raft_consensus.cc:2454] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:36.996454 22784 leader_election.cc:302] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:36.996600 24279 raft_consensus.cc:2781] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:36.996687 24279 raft_consensus.cc:686] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:36.996796 24279 consensus_queue.cc:237] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:36.998171 22441 catalog_manager.cc:5484] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:37.167049 24015 raft_consensus.cc:482] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:37.167160 24015 raft_consensus.cc:504] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:37.167484 24015 leader_election.cc:288] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:37.167783 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eecae5d013f543bf8eed1f2488757115" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:37.167819 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eecae5d013f543bf8eed1f2488757115" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:37.167908 22848 raft_consensus.cc:2454] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:37.167927 22584 raft_consensus.cc:2454] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:37.168161 22915 leader_election.cc:302] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:37.168314 24015 raft_consensus.cc:2781] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:37.168373 24015 raft_consensus.cc:482] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:37.168397 24015 raft_consensus.cc:3037] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:37.169132 24015 raft_consensus.cc:504] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:37.169394 24015 leader_election.cc:288] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:37.169696 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eecae5d013f543bf8eed1f2488757115" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:37.169803 22848 raft_consensus.cc:3037] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:37.169773 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eecae5d013f543bf8eed1f2488757115" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:37.169874 22584 raft_consensus.cc:3037] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:37.170748 22848 raft_consensus.cc:2454] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:37.170746 22584 raft_consensus.cc:2454] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:37.171010 22915 leader_election.cc:302] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:37.171165 24015 raft_consensus.cc:2781] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:37.171250 24015 raft_consensus.cc:686] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:37.171382 24015 consensus_queue.cc:237] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:37.173151 22441 catalog_manager.cc:5484] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:37.278811 24315 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:37.279022 24315 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:37.281795 24315 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_12'
I20240502 07:58:37.351680 24184 consensus_queue.cc:1035] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:37.361493 22584 raft_consensus.cc:1261] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:37.362300 22715 raft_consensus.cc:1261] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:37.362677 24279 consensus_queue.cc:1035] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:37.362816 24029 consensus_queue.cc:1035] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:37.363605 22717 raft_consensus.cc:1261] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:37.364814 22584 raft_consensus.cc:1261] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:37.365168 24015 consensus_queue.cc:1035] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:37.366291 24311 consensus_queue.cc:1035] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:37.367523 22848 raft_consensus.cc:1261] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:37.372143 24015 consensus_queue.cc:1035] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 111.059 ms
time per row: 0.0555295 ms
I20240502 07:58:37.433404 22440 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_13"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:37.438488 22827 tablet_service.cc:1467] Processing CreateTablet for tablet f1fa312fe8e44d23970c5e3ba98c2c58 (DEFAULT_TABLE table=rebalance_test_table_13 [id=2882412f9f504bfc93c45c294c9ace87]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.438760 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f1fa312fe8e44d23970c5e3ba98c2c58. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.439802 22566 tablet_service.cc:1467] Processing CreateTablet for tablet c53e8b3bd774466dbafe4994472975bc (DEFAULT_TABLE table=rebalance_test_table_13 [id=2882412f9f504bfc93c45c294c9ace87]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.440032 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c53e8b3bd774466dbafe4994472975bc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.439978 22696 tablet_service.cc:1467] Processing CreateTablet for tablet f1fa312fe8e44d23970c5e3ba98c2c58 (DEFAULT_TABLE table=rebalance_test_table_13 [id=2882412f9f504bfc93c45c294c9ace87]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.440209 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f1fa312fe8e44d23970c5e3ba98c2c58. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.440903 22959 tablet_service.cc:1467] Processing CreateTablet for tablet f1fa312fe8e44d23970c5e3ba98c2c58 (DEFAULT_TABLE table=rebalance_test_table_13 [id=2882412f9f504bfc93c45c294c9ace87]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.441133 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f1fa312fe8e44d23970c5e3ba98c2c58. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.441870 24301 tablet_bootstrap.cc:492] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:37.442087 22697 tablet_service.cc:1467] Processing CreateTablet for tablet a98f26c1f53f403f9a9f6ae994cd3caa (DEFAULT_TABLE table=rebalance_test_table_13 [id=2882412f9f504bfc93c45c294c9ace87]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.442291 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a98f26c1f53f403f9a9f6ae994cd3caa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.442765 24301 tablet_bootstrap.cc:654] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.443265 22958 tablet_service.cc:1467] Processing CreateTablet for tablet a98f26c1f53f403f9a9f6ae994cd3caa (DEFAULT_TABLE table=rebalance_test_table_13 [id=2882412f9f504bfc93c45c294c9ace87]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.443493 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a98f26c1f53f403f9a9f6ae994cd3caa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.443997 22695 tablet_service.cc:1467] Processing CreateTablet for tablet c53e8b3bd774466dbafe4994472975bc (DEFAULT_TABLE table=rebalance_test_table_13 [id=2882412f9f504bfc93c45c294c9ace87]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.444164 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c53e8b3bd774466dbafe4994472975bc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.444761 24298 tablet_bootstrap.cc:492] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:37.445389 24299 tablet_bootstrap.cc:492] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:37.445413 24298 tablet_bootstrap.cc:654] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.445796 24302 tablet_bootstrap.cc:492] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:37.446045 24299 tablet_bootstrap.cc:654] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.446235 24298 tablet_bootstrap.cc:492] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:37.446309 24298 ts_tablet_manager.cc:1367] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:37.446350 24302 tablet_bootstrap.cc:654] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.446688 24298 raft_consensus.cc:348] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.446816 24298 raft_consensus.cc:374] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.446849 24298 raft_consensus.cc:729] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.446965 24298 consensus_queue.cc:260] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.447221 24298 ts_tablet_manager.cc:1398] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:37.447469 24302 tablet_bootstrap.cc:492] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:37.447535 24302 ts_tablet_manager.cc:1367] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:37.447598 24299 tablet_bootstrap.cc:492] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:37.447669 24299 ts_tablet_manager.cc:1367] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:37.447814 24302 raft_consensus.cc:348] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.447876 24302 raft_consensus.cc:374] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.447894 24302 raft_consensus.cc:729] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.447984 24302 consensus_queue.cc:260] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.448007 24299 raft_consensus.cc:348] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.448091 24299 raft_consensus.cc:374] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.448122 24299 raft_consensus.cc:729] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.448206 24302 ts_tablet_manager.cc:1398] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:37.448242 24299 consensus_queue.cc:260] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.448346 24302 tablet_bootstrap.cc:492] T a98f26c1f53f403f9a9f6ae994cd3caa P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:37.448509 24299 ts_tablet_manager.cc:1398] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:37.448671 24299 tablet_bootstrap.cc:492] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:37.449090 24302 tablet_bootstrap.cc:654] T a98f26c1f53f403f9a9f6ae994cd3caa P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.449455 24299 tablet_bootstrap.cc:654] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.450423 24299 tablet_bootstrap.cc:492] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:37.450495 24299 ts_tablet_manager.cc:1367] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:37.450534 22826 tablet_service.cc:1467] Processing CreateTablet for tablet c53e8b3bd774466dbafe4994472975bc (DEFAULT_TABLE table=rebalance_test_table_13 [id=2882412f9f504bfc93c45c294c9ace87]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.450763 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c53e8b3bd774466dbafe4994472975bc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.450790 22828 tablet_service.cc:1467] Processing CreateTablet for tablet a98f26c1f53f403f9a9f6ae994cd3caa (DEFAULT_TABLE table=rebalance_test_table_13 [id=2882412f9f504bfc93c45c294c9ace87]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.450887 24299 raft_consensus.cc:348] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.450968 24299 raft_consensus.cc:374] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.450994 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a98f26c1f53f403f9a9f6ae994cd3caa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.451005 24299 raft_consensus.cc:729] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.451133 24299 consensus_queue.cc:260] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.451407 24299 ts_tablet_manager.cc:1398] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:37.451548 24299 tablet_bootstrap.cc:492] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:37.452396 24299 tablet_bootstrap.cc:654] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.452760 24298 tablet_bootstrap.cc:492] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:37.453964 24298 tablet_bootstrap.cc:654] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.455019 24302 tablet_bootstrap.cc:492] T a98f26c1f53f403f9a9f6ae994cd3caa P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:37.455099 24302 ts_tablet_manager.cc:1367] T a98f26c1f53f403f9a9f6ae994cd3caa P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.001s
I20240502 07:58:37.455152 24298 tablet_bootstrap.cc:492] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:37.455227 24298 ts_tablet_manager.cc:1367] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:37.455596 24298 raft_consensus.cc:348] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:37.455682 24298 raft_consensus.cc:374] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.455718 24298 raft_consensus.cc:729] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.455724 24302 raft_consensus.cc:348] T a98f26c1f53f403f9a9f6ae994cd3caa P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.455803 24302 raft_consensus.cc:374] T a98f26c1f53f403f9a9f6ae994cd3caa P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.455808 24298 consensus_queue.cc:260] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:37.455838 24302 raft_consensus.cc:729] T a98f26c1f53f403f9a9f6ae994cd3caa P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.456048 24298 ts_tablet_manager.cc:1398] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:37.456037 24302 consensus_queue.cc:260] T a98f26c1f53f403f9a9f6ae994cd3caa P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.456197 24298 tablet_bootstrap.cc:492] T a98f26c1f53f403f9a9f6ae994cd3caa P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:37.456344 24302 ts_tablet_manager.cc:1398] T a98f26c1f53f403f9a9f6ae994cd3caa P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:37.456928 24298 tablet_bootstrap.cc:654] T a98f26c1f53f403f9a9f6ae994cd3caa P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.457253 24301 tablet_bootstrap.cc:492] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:37.457321 24301 ts_tablet_manager.cc:1367] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.015s user 0.002s sys 0.000s
I20240502 07:58:37.457315 24299 tablet_bootstrap.cc:492] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:37.457381 24299 ts_tablet_manager.cc:1367] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:58:37.457727 24301 raft_consensus.cc:348] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:37.457834 24301 raft_consensus.cc:374] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.457865 24301 raft_consensus.cc:729] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.457984 24301 consensus_queue.cc:260] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:37.458092 24298 tablet_bootstrap.cc:492] T a98f26c1f53f403f9a9f6ae994cd3caa P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:37.458166 24298 ts_tablet_manager.cc:1367] T a98f26c1f53f403f9a9f6ae994cd3caa P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:37.458242 24301 ts_tablet_manager.cc:1398] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:37.458534 24298 raft_consensus.cc:348] T a98f26c1f53f403f9a9f6ae994cd3caa P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.458616 24298 raft_consensus.cc:374] T a98f26c1f53f403f9a9f6ae994cd3caa P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.458645 24298 raft_consensus.cc:729] T a98f26c1f53f403f9a9f6ae994cd3caa P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.458753 24298 consensus_queue.cc:260] T a98f26c1f53f403f9a9f6ae994cd3caa P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.459007 24298 ts_tablet_manager.cc:1398] T a98f26c1f53f403f9a9f6ae994cd3caa P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:37.459062 24299 raft_consensus.cc:348] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:37.459141 24299 raft_consensus.cc:374] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.459178 24299 raft_consensus.cc:729] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.459286 24299 consensus_queue.cc:260] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:37.460049 24299 ts_tablet_manager.cc:1398] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:37.460918 24015 raft_consensus.cc:482] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:37.460992 24015 raft_consensus.cc:504] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.461320 24015 leader_election.cc:288] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:37.461658 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f1fa312fe8e44d23970c5e3ba98c2c58" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:37.461766 22715 raft_consensus.cc:2454] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:37.462013 22912 leader_election.cc:302] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:37.462399 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f1fa312fe8e44d23970c5e3ba98c2c58" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:37.462510 22848 raft_consensus.cc:2454] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:37.462695 24015 raft_consensus.cc:2781] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:37.462755 24015 raft_consensus.cc:482] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:37.462797 24015 raft_consensus.cc:3037] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:37.463475 24015 raft_consensus.cc:504] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.463811 24015 leader_election.cc:288] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:37.464021 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f1fa312fe8e44d23970c5e3ba98c2c58" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:37.464064 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f1fa312fe8e44d23970c5e3ba98c2c58" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:37.464110 22848 raft_consensus.cc:3037] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:37.464133 22715 raft_consensus.cc:3037] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:37.464745 22848 raft_consensus.cc:2454] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:37.464774 22715 raft_consensus.cc:2454] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:37.464948 22915 leader_election.cc:302] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:37.465101 24015 raft_consensus.cc:2781] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:37.465170 24015 raft_consensus.cc:686] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:37.465279 24015 consensus_queue.cc:237] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.466660 22444 catalog_manager.cc:5484] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:37.484966 24184 raft_consensus.cc:482] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:37.485054 24184 raft_consensus.cc:504] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:37.485347 24184 leader_election.cc:288] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:37.485613 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c53e8b3bd774466dbafe4994472975bc" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:37.485631 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c53e8b3bd774466dbafe4994472975bc" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:37.485715 22848 raft_consensus.cc:2454] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:37.485715 22715 raft_consensus.cc:2454] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:37.485949 22519 leader_election.cc:302] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:37.486110 24184 raft_consensus.cc:2781] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:37.486160 24184 raft_consensus.cc:482] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:37.486186 24184 raft_consensus.cc:3037] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:37.486866 24184 raft_consensus.cc:504] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:37.487128 24184 leader_election.cc:288] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:37.487386 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c53e8b3bd774466dbafe4994472975bc" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:37.487386 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c53e8b3bd774466dbafe4994472975bc" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:37.487486 22715 raft_consensus.cc:3037] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:37.487486 22848 raft_consensus.cc:3037] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:37.488237 22848 raft_consensus.cc:2454] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:37.488237 22715 raft_consensus.cc:2454] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:37.488490 22522 leader_election.cc:302] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:37.488636 24184 raft_consensus.cc:2781] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:37.488729 24184 raft_consensus.cc:686] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:37.488837 24184 consensus_queue.cc:237] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:37.490351 22444 catalog_manager.cc:5484] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:37.601548 24132 raft_consensus.cc:482] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:37.601655 24132 raft_consensus.cc:504] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.601945 24132 leader_election.cc:288] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:37.602205 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a98f26c1f53f403f9a9f6ae994cd3caa" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:37.602286 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a98f26c1f53f403f9a9f6ae994cd3caa" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:37.602355 22848 raft_consensus.cc:2454] T a98f26c1f53f403f9a9f6ae994cd3caa P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:37.602433 22979 raft_consensus.cc:2454] T a98f26c1f53f403f9a9f6ae994cd3caa P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:37.602644 22653 leader_election.cc:302] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:37.602784 24132 raft_consensus.cc:2781] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:37.602838 24132 raft_consensus.cc:482] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:37.602864 24132 raft_consensus.cc:3037] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:37.603638 24132 raft_consensus.cc:504] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.603911 24132 leader_election.cc:288] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:37.604105 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a98f26c1f53f403f9a9f6ae994cd3caa" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:37.604226 22848 raft_consensus.cc:3037] T a98f26c1f53f403f9a9f6ae994cd3caa P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:37.604197 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a98f26c1f53f403f9a9f6ae994cd3caa" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:37.604305 22979 raft_consensus.cc:3037] T a98f26c1f53f403f9a9f6ae994cd3caa P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:37.604934 22848 raft_consensus.cc:2454] T a98f26c1f53f403f9a9f6ae994cd3caa P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:37.605127 22979 raft_consensus.cc:2454] T a98f26c1f53f403f9a9f6ae994cd3caa P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:37.605167 22653 leader_election.cc:302] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:37.605376 24132 raft_consensus.cc:2781] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:37.605443 24132 raft_consensus.cc:686] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:37.605571 24132 consensus_queue.cc:237] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.607192 22444 catalog_manager.cc:5484] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:37.748915 24366 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:37.749190 24366 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:37.752786 24366 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_13'
I20240502 07:58:37.835352 22717 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:37.836177 22848 raft_consensus.cc:1261] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:37.836557 24184 consensus_queue.cc:1035] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:37.837785 22847 raft_consensus.cc:1261] T a98f26c1f53f403f9a9f6ae994cd3caa P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:37.838241 24132 consensus_queue.cc:1035] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:37.839300 22848 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:37.840871 22717 raft_consensus.cc:1261] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:37.841876 24015 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:37.842963 22979 raft_consensus.cc:1261] T a98f26c1f53f403f9a9f6ae994cd3caa P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:37.843135 24015 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:37.844476 24311 consensus_queue.cc:1035] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:37.844589 24132 consensus_queue.cc:1035] T a98f26c1f53f403f9a9f6ae994cd3caa P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 111.789 ms
time per row: 0.0558947 ms
I20240502 07:58:37.902863 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_14"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:37.907945 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 1d382df270004c128dc5718be32532ba (DEFAULT_TABLE table=rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.908066 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 1d382df270004c128dc5718be32532ba (DEFAULT_TABLE table=rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.908226 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1d382df270004c128dc5718be32532ba. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.908295 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1d382df270004c128dc5718be32532ba. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.908710 22697 tablet_service.cc:1467] Processing CreateTablet for tablet c8c7521bb8414364b927eed84be37621 (DEFAULT_TABLE table=rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.908903 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c8c7521bb8414364b927eed84be37621. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.909041 22958 tablet_service.cc:1467] Processing CreateTablet for tablet 1d382df270004c128dc5718be32532ba (DEFAULT_TABLE table=rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.909256 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1d382df270004c128dc5718be32532ba. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.910391 24301 tablet_bootstrap.cc:492] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:37.911088 24302 tablet_bootstrap.cc:492] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:37.911195 24301 tablet_bootstrap.cc:654] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.911258 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 848fc8384054446aa75a05a52488ea57 (DEFAULT_TABLE table=rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.911476 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 848fc8384054446aa75a05a52488ea57. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.911808 24302 tablet_bootstrap.cc:654] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.912149 22564 tablet_service.cc:1467] Processing CreateTablet for tablet c8c7521bb8414364b927eed84be37621 (DEFAULT_TABLE table=rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.912391 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c8c7521bb8414364b927eed84be37621. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.912835 24299 tablet_bootstrap.cc:492] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:37.913548 24299 tablet_bootstrap.cc:654] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.913549 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 848fc8384054446aa75a05a52488ea57 (DEFAULT_TABLE table=rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.913715 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 848fc8384054446aa75a05a52488ea57. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.914171 22957 tablet_service.cc:1467] Processing CreateTablet for tablet c8c7521bb8414364b927eed84be37621 (DEFAULT_TABLE table=rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.914355 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c8c7521bb8414364b927eed84be37621. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.914705 24299 tablet_bootstrap.cc:492] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:37.914767 24299 ts_tablet_manager.cc:1367] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:37.915140 24299 raft_consensus.cc:348] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:37.915251 24299 raft_consensus.cc:374] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.915287 24299 raft_consensus.cc:729] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.915402 24299 consensus_queue.cc:260] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:37.915664 24299 ts_tablet_manager.cc:1398] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:37.915684 24301 tablet_bootstrap.cc:492] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:37.915745 24301 ts_tablet_manager.cc:1367] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:37.915817 24299 tablet_bootstrap.cc:492] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:37.916086 24301 raft_consensus.cc:348] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:37.916177 24301 raft_consensus.cc:374] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.916206 24301 raft_consensus.cc:729] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.916323 24301 consensus_queue.cc:260] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:37.916554 24299 tablet_bootstrap.cc:654] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.916594 24301 ts_tablet_manager.cc:1398] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:37.916762 24301 tablet_bootstrap.cc:492] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:37.917351 24301 tablet_bootstrap.cc:654] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.917366 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 848fc8384054446aa75a05a52488ea57 (DEFAULT_TABLE table=rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:37.917554 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 848fc8384054446aa75a05a52488ea57. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:37.918107 24301 tablet_bootstrap.cc:492] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:37.918155 24301 ts_tablet_manager.cc:1367] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:37.918465 24301 raft_consensus.cc:348] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.918537 24301 raft_consensus.cc:374] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.918555 24301 raft_consensus.cc:729] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.918640 24301 consensus_queue.cc:260] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.918859 24301 ts_tablet_manager.cc:1398] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:37.918977 24301 tablet_bootstrap.cc:492] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:37.919368 24298 tablet_bootstrap.cc:492] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:37.919610 24301 tablet_bootstrap.cc:654] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.920140 24298 tablet_bootstrap.cc:654] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.920509 24301 tablet_bootstrap.cc:492] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:37.920583 24301 ts_tablet_manager.cc:1367] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:37.920967 24301 raft_consensus.cc:348] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:37.921043 24298 tablet_bootstrap.cc:492] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:37.921053 24301 raft_consensus.cc:374] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.921083 24301 raft_consensus.cc:729] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.921115 24298 ts_tablet_manager.cc:1367] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:37.921175 24301 consensus_queue.cc:260] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:37.921420 24301 ts_tablet_manager.cc:1398] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:37.921480 24298 raft_consensus.cc:348] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:37.921576 24298 raft_consensus.cc:374] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.921612 24298 raft_consensus.cc:729] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.921712 24298 consensus_queue.cc:260] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:37.921938 24298 ts_tablet_manager.cc:1398] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:37.923323 24299 tablet_bootstrap.cc:492] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:37.923404 24299 ts_tablet_manager.cc:1367] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.000s
I20240502 07:58:37.923568 24302 tablet_bootstrap.cc:492] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:37.923632 24302 ts_tablet_manager.cc:1367] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.013s user 0.000s sys 0.001s
I20240502 07:58:37.923738 24299 raft_consensus.cc:348] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.923799 24299 raft_consensus.cc:374] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.923816 24299 raft_consensus.cc:729] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.923895 24299 consensus_queue.cc:260] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.923931 24302 raft_consensus.cc:348] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:37.924008 24302 raft_consensus.cc:374] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.924036 24302 raft_consensus.cc:729] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.924100 24299 ts_tablet_manager.cc:1398] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:37.924124 24302 consensus_queue.cc:260] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:37.924315 24302 ts_tablet_manager.cc:1398] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:37.924430 24302 tablet_bootstrap.cc:492] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:37.926522 24302 tablet_bootstrap.cc:654] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.927403 24302 tablet_bootstrap.cc:492] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:37.927472 24302 ts_tablet_manager.cc:1367] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:37.927803 24302 raft_consensus.cc:348] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:37.927886 24302 raft_consensus.cc:374] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.927917 24302 raft_consensus.cc:729] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.928017 24302 consensus_queue.cc:260] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:37.928236 24302 ts_tablet_manager.cc:1398] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:37.928378 24302 tablet_bootstrap.cc:492] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:37.929143 24302 tablet_bootstrap.cc:654] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:37.930075 24302 tablet_bootstrap.cc:492] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:37.930151 24302 ts_tablet_manager.cc:1367] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:37.930470 24302 raft_consensus.cc:348] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.930548 24302 raft_consensus.cc:374] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:37.930600 24302 raft_consensus.cc:729] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:37.930719 24302 consensus_queue.cc:260] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:37.930965 24302 ts_tablet_manager.cc:1398] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:37.932858 24015 raft_consensus.cc:482] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:37.932930 24015 raft_consensus.cc:504] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:37.933235 24015 leader_election.cc:288] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:37.933526 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1d382df270004c128dc5718be32532ba" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:37.933605 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1d382df270004c128dc5718be32532ba" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:37.933658 22586 raft_consensus.cc:2454] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:37.933701 22717 raft_consensus.cc:2454] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:37.933892 22915 leader_election.cc:302] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:37.934095 24015 raft_consensus.cc:2781] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:37.934146 24015 raft_consensus.cc:482] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:37.934183 24015 raft_consensus.cc:3037] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:37.934870 24015 raft_consensus.cc:504] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:37.935154 24015 leader_election.cc:288] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:37.935418 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1d382df270004c128dc5718be32532ba" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:37.935477 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1d382df270004c128dc5718be32532ba" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:37.935518 22717 raft_consensus.cc:3037] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:37.935566 22586 raft_consensus.cc:3037] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:37.936372 22586 raft_consensus.cc:2454] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:37.936378 22717 raft_consensus.cc:2454] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:37.936635 22915 leader_election.cc:302] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:37.936812 24015 raft_consensus.cc:2781] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:37.936904 24015 raft_consensus.cc:686] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:37.937016 24015 consensus_queue.cc:237] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:37.938292 22444 catalog_manager.cc:5484] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:38.036757 24184 raft_consensus.cc:482] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:38.036860 24184 raft_consensus.cc:504] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.037163 24184 leader_election.cc:288] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:38.037453 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c8c7521bb8414364b927eed84be37621" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:38.037464 22977 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c8c7521bb8414364b927eed84be37621" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:38.037564 22977 raft_consensus.cc:2454] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:38.037578 22717 raft_consensus.cc:2454] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:38.037817 22520 leader_election.cc:302] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:38.038007 24184 raft_consensus.cc:2781] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:38.038055 24184 raft_consensus.cc:482] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:38.038079 24184 raft_consensus.cc:3037] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.038831 24184 raft_consensus.cc:504] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.039121 24184 leader_election.cc:288] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:38.039394 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c8c7521bb8414364b927eed84be37621" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:38.039426 22977 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c8c7521bb8414364b927eed84be37621" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:38.039495 22717 raft_consensus.cc:3037] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.039561 22977 raft_consensus.cc:3037] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.040177 22717 raft_consensus.cc:2454] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:38.040249 22977 raft_consensus.cc:2454] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:38.040465 22519 leader_election.cc:302] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:38.040618 24184 raft_consensus.cc:2781] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:38.040726 24184 raft_consensus.cc:686] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:38.040859 24184 consensus_queue.cc:237] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.042557 22444 catalog_manager.cc:5484] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:38.056731 24029 raft_consensus.cc:482] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:38.056823 24029 raft_consensus.cc:504] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:38.057125 24029 leader_election.cc:288] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:38.057397 22977 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "848fc8384054446aa75a05a52488ea57" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:38.057394 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "848fc8384054446aa75a05a52488ea57" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:38.057488 22977 raft_consensus.cc:2454] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:38.057498 22584 raft_consensus.cc:2454] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:38.057740 22782 leader_election.cc:302] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:38.057896 24029 raft_consensus.cc:2781] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:38.057948 24029 raft_consensus.cc:482] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:38.057996 24029 raft_consensus.cc:3037] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.058724 24029 raft_consensus.cc:504] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:38.059015 24029 leader_election.cc:288] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:38.059247 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "848fc8384054446aa75a05a52488ea57" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:38.059298 22977 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "848fc8384054446aa75a05a52488ea57" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:38.059336 22584 raft_consensus.cc:3037] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.059398 22977 raft_consensus.cc:3037] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.060009 22584 raft_consensus.cc:2454] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:38.060024 22977 raft_consensus.cc:2454] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:38.060261 22782 leader_election.cc:302] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:38.060420 24029 raft_consensus.cc:2781] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:38.060492 24029 raft_consensus.cc:686] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:38.060616 24029 consensus_queue.cc:237] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:38.062165 22444 catalog_manager.cc:5484] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } }
W20240502 07:58:38.170809 24416 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:38.170993 24416 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:38.174249 24416 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_14'
I20240502 07:58:38.243033 22977 raft_consensus.cc:1261] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:38.243427 24184 consensus_queue.cc:1035] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:38.244112 22717 raft_consensus.cc:1261] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:38.245338 22979 raft_consensus.cc:1261] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:38.245730 24029 consensus_queue.cc:1035] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:38.246944 22584 raft_consensus.cc:1261] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:38.247179 24311 consensus_queue.cc:1035] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:38.247984 24029 consensus_queue.cc:1035] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:38.249269 22586 raft_consensus.cc:1261] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:38.249604 24081 consensus_queue.cc:1035] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:38.253873 22715 raft_consensus.cc:1261] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:38.254971 24081 consensus_queue.cc:1035] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:38.267124 24268 mvcc.cc:205] Tried to move back new op lower bound from 7023151997923319808 to 7023151997177470976. Current Snapshot: MvccSnapshot[applied={T|T < 7023151997923319808}]
I20240502 07:58:38.287159 24272 mvcc.cc:205] Tried to move back new op lower bound from 7023151997914341376 to 7023151997096361984. Current Snapshot: MvccSnapshot[applied={T|T < 7023151997914341376}]
I20240502 07:58:38.291267 24267 mvcc.cc:205] Tried to move back new op lower bound from 7023151997923319808 to 7023151997177470976. Current Snapshot: MvccSnapshot[applied={T|T < 7023151997923319808}]
INSERT report
rows total: 2000
time total: 94.3819 ms
time per row: 0.0471909 ms
I20240502 07:58:38.308629 22438 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_15"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:38.313655 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 9e5fe86b44744df9b2e4d430fdec716f (DEFAULT_TABLE table=rebalance_test_table_15 [id=4b1b337aafc04dc69a2e37f348a5652a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.313935 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9e5fe86b44744df9b2e4d430fdec716f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.314023 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 44f7d6f2a16d4c6b88526760cacc97a5 (DEFAULT_TABLE table=rebalance_test_table_15 [id=4b1b337aafc04dc69a2e37f348a5652a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.314267 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 44f7d6f2a16d4c6b88526760cacc97a5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.314626 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 539cf5d4596f481cb03ab4d13a19198c (DEFAULT_TABLE table=rebalance_test_table_15 [id=4b1b337aafc04dc69a2e37f348a5652a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.314819 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 539cf5d4596f481cb03ab4d13a19198c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.315809 24302 tablet_bootstrap.cc:492] T 44f7d6f2a16d4c6b88526760cacc97a5 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:38.315980 24298 tablet_bootstrap.cc:492] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:38.316427 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 9e5fe86b44744df9b2e4d430fdec716f (DEFAULT_TABLE table=rebalance_test_table_15 [id=4b1b337aafc04dc69a2e37f348a5652a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.316694 24302 tablet_bootstrap.cc:654] T 44f7d6f2a16d4c6b88526760cacc97a5 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.316816 24298 tablet_bootstrap.cc:654] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.316998 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 44f7d6f2a16d4c6b88526760cacc97a5 (DEFAULT_TABLE table=rebalance_test_table_15 [id=4b1b337aafc04dc69a2e37f348a5652a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.316659 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9e5fe86b44744df9b2e4d430fdec716f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.317288 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 44f7d6f2a16d4c6b88526760cacc97a5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.317797 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 539cf5d4596f481cb03ab4d13a19198c (DEFAULT_TABLE table=rebalance_test_table_15 [id=4b1b337aafc04dc69a2e37f348a5652a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.318009 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 539cf5d4596f481cb03ab4d13a19198c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.318306 24298 tablet_bootstrap.cc:492] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:38.318375 24298 ts_tablet_manager.cc:1367] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:38.318778 24298 raft_consensus.cc:348] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:38.318878 24298 raft_consensus.cc:374] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.318912 24298 raft_consensus.cc:729] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.319047 24298 consensus_queue.cc:260] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:38.319164 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 9e5fe86b44744df9b2e4d430fdec716f (DEFAULT_TABLE table=rebalance_test_table_15 [id=4b1b337aafc04dc69a2e37f348a5652a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.319204 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 539cf5d4596f481cb03ab4d13a19198c (DEFAULT_TABLE table=rebalance_test_table_15 [id=4b1b337aafc04dc69a2e37f348a5652a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.319319 24298 ts_tablet_manager.cc:1398] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:38.319428 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9e5fe86b44744df9b2e4d430fdec716f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.319741 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 539cf5d4596f481cb03ab4d13a19198c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.319818 24301 tablet_bootstrap.cc:492] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:38.320591 24301 tablet_bootstrap.cc:654] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.321379 24299 tablet_bootstrap.cc:492] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:38.321463 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 44f7d6f2a16d4c6b88526760cacc97a5 (DEFAULT_TABLE table=rebalance_test_table_15 [id=4b1b337aafc04dc69a2e37f348a5652a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.321632 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 44f7d6f2a16d4c6b88526760cacc97a5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.322125 24299 tablet_bootstrap.cc:654] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.322770 24301 tablet_bootstrap.cc:492] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:38.322840 24301 ts_tablet_manager.cc:1367] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.001s
I20240502 07:58:38.323172 24301 raft_consensus.cc:348] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:38.323308 24301 raft_consensus.cc:374] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.323343 24301 raft_consensus.cc:729] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.323474 24301 consensus_queue.cc:260] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:38.323697 24298 tablet_bootstrap.cc:492] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:38.323732 24301 ts_tablet_manager.cc:1398] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:38.323933 24301 tablet_bootstrap.cc:492] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:38.324549 24298 tablet_bootstrap.cc:654] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.324769 24301 tablet_bootstrap.cc:654] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.325518 24299 tablet_bootstrap.cc:492] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:38.325603 24299 ts_tablet_manager.cc:1367] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:38.325657 24302 tablet_bootstrap.cc:492] T 44f7d6f2a16d4c6b88526760cacc97a5 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:38.325726 24302 ts_tablet_manager.cc:1367] T 44f7d6f2a16d4c6b88526760cacc97a5 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.010s user 0.000s sys 0.001s
I20240502 07:58:38.325901 24301 tablet_bootstrap.cc:492] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:38.326013 24301 ts_tablet_manager.cc:1367] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.002s
I20240502 07:58:38.326056 24299 raft_consensus.cc:348] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.326172 24299 raft_consensus.cc:374] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.326135 24302 raft_consensus.cc:348] T 44f7d6f2a16d4c6b88526760cacc97a5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:38.326223 24299 raft_consensus.cc:729] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.326257 24302 raft_consensus.cc:374] T 44f7d6f2a16d4c6b88526760cacc97a5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.326280 24302 raft_consensus.cc:729] T 44f7d6f2a16d4c6b88526760cacc97a5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.326371 24299 consensus_queue.cc:260] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.326390 24302 consensus_queue.cc:260] T 44f7d6f2a16d4c6b88526760cacc97a5 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:38.326642 24302 ts_tablet_manager.cc:1398] T 44f7d6f2a16d4c6b88526760cacc97a5 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:38.326637 24299 ts_tablet_manager.cc:1398] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:38.326762 24302 tablet_bootstrap.cc:492] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:38.326797 24299 tablet_bootstrap.cc:492] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:38.326771 24301 raft_consensus.cc:348] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:38.326947 24301 raft_consensus.cc:374] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.327011 24301 raft_consensus.cc:729] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.327183 24301 consensus_queue.cc:260] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:38.327651 24299 tablet_bootstrap.cc:654] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.327728 24301 ts_tablet_manager.cc:1398] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:38.328029 24302 tablet_bootstrap.cc:654] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.328702 24298 tablet_bootstrap.cc:492] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:38.328779 24298 ts_tablet_manager.cc:1367] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.001s
I20240502 07:58:38.329066 24302 tablet_bootstrap.cc:492] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:38.329073 24298 raft_consensus.cc:348] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.329141 24302 ts_tablet_manager.cc:1367] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:38.329149 24298 raft_consensus.cc:374] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.329176 24298 raft_consensus.cc:729] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.329290 24298 consensus_queue.cc:260] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.329560 24298 ts_tablet_manager.cc:1398] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:38.329690 24298 tablet_bootstrap.cc:492] T 44f7d6f2a16d4c6b88526760cacc97a5 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:38.330400 24298 tablet_bootstrap.cc:654] T 44f7d6f2a16d4c6b88526760cacc97a5 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.330920 24453 raft_consensus.cc:482] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:38.331009 24453 raft_consensus.cc:504] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:38.331315 24453 leader_election.cc:288] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:38.331810 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9e5fe86b44744df9b2e4d430fdec716f" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:38.331868 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9e5fe86b44744df9b2e4d430fdec716f" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:38.331929 22845 raft_consensus.cc:2454] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:38.332206 22522 leader_election.cc:302] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
W20240502 07:58:38.332372 22519 leader_election.cc:341] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): Illegal state: must be running to vote when last-logged opid is not known
I20240502 07:58:38.332439 24453 raft_consensus.cc:2781] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:38.332480 24453 raft_consensus.cc:482] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:38.332513 24453 raft_consensus.cc:3037] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.332952 24299 tablet_bootstrap.cc:492] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:38.333040 24299 ts_tablet_manager.cc:1367] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:58:38.331466 24302 raft_consensus.cc:348] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.333345 24302 raft_consensus.cc:374] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.333406 24302 raft_consensus.cc:729] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.333691 24453 raft_consensus.cc:504] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:38.333532 24302 consensus_queue.cc:260] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.333868 24299 raft_consensus.cc:348] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:38.333956 24299 raft_consensus.cc:374] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.334008 24453 leader_election.cc:288] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:38.333986 24299 raft_consensus.cc:729] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.334218 24299 consensus_queue.cc:260] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:38.334290 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9e5fe86b44744df9b2e4d430fdec716f" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:38.334357 22715 raft_consensus.cc:3037] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.334554 24302 ts_tablet_manager.cc:1398] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:38.334614 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9e5fe86b44744df9b2e4d430fdec716f" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:38.334702 22845 raft_consensus.cc:3037] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.334820 24299 ts_tablet_manager.cc:1398] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:38.335386 22715 raft_consensus.cc:2454] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:38.335458 24298 tablet_bootstrap.cc:492] T 44f7d6f2a16d4c6b88526760cacc97a5 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:38.335477 22845 raft_consensus.cc:2454] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:38.335523 24298 ts_tablet_manager.cc:1367] T 44f7d6f2a16d4c6b88526760cacc97a5 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:58:38.335736 22522 leader_election.cc:302] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:38.335911 24453 raft_consensus.cc:2781] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:38.335950 24298 raft_consensus.cc:348] T 44f7d6f2a16d4c6b88526760cacc97a5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:38.336030 24298 raft_consensus.cc:374] T 44f7d6f2a16d4c6b88526760cacc97a5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.336067 24298 raft_consensus.cc:729] T 44f7d6f2a16d4c6b88526760cacc97a5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.336164 24298 consensus_queue.cc:260] T 44f7d6f2a16d4c6b88526760cacc97a5 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:38.336391 24298 ts_tablet_manager.cc:1398] T 44f7d6f2a16d4c6b88526760cacc97a5 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:38.337249 24453 raft_consensus.cc:686] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:38.337392 24453 consensus_queue.cc:237] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:38.339809 22441 catalog_manager.cc:5484] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:38.402765 24453 raft_consensus.cc:482] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:38.402881 24453 raft_consensus.cc:504] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:38.403224 24453 leader_election.cc:288] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:38.403488 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "44f7d6f2a16d4c6b88526760cacc97a5" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:38.403616 22979 raft_consensus.cc:2454] T 44f7d6f2a16d4c6b88526760cacc97a5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:38.403625 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "44f7d6f2a16d4c6b88526760cacc97a5" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:38.403750 22845 raft_consensus.cc:2454] T 44f7d6f2a16d4c6b88526760cacc97a5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:38.403903 22520 leader_election.cc:302] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:38.404088 24453 raft_consensus.cc:2781] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:38.404150 24453 raft_consensus.cc:482] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:38.404202 24453 raft_consensus.cc:3037] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.405107 24453 raft_consensus.cc:504] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:38.405401 24453 leader_election.cc:288] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:38.405643 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "44f7d6f2a16d4c6b88526760cacc97a5" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:38.405730 22979 raft_consensus.cc:3037] T 44f7d6f2a16d4c6b88526760cacc97a5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.406440 22979 raft_consensus.cc:2454] T 44f7d6f2a16d4c6b88526760cacc97a5 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:38.406641 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "44f7d6f2a16d4c6b88526760cacc97a5" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:38.406735 22845 raft_consensus.cc:3037] T 44f7d6f2a16d4c6b88526760cacc97a5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.406719 22520 leader_election.cc:302] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:38.406847 24453 raft_consensus.cc:2781] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:38.406958 24453 raft_consensus.cc:686] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:38.407090 24453 consensus_queue.cc:237] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:38.407593 22845 raft_consensus.cc:2454] T 44f7d6f2a16d4c6b88526760cacc97a5 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:38.409090 22441 catalog_manager.cc:5484] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:38.503028 24280 raft_consensus.cc:482] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:38.503134 24280 raft_consensus.cc:504] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.503508 24280 leader_election.cc:288] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:38.503806 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "539cf5d4596f481cb03ab4d13a19198c" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:38.503835 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "539cf5d4596f481cb03ab4d13a19198c" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:38.503922 22845 raft_consensus.cc:2454] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:38.503937 22979 raft_consensus.cc:2454] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:38.504177 22653 leader_election.cc:302] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:38.504338 24280 raft_consensus.cc:2781] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:38.504400 24280 raft_consensus.cc:482] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:38.504427 24280 raft_consensus.cc:3037] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.505230 24280 raft_consensus.cc:504] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.505519 24280 leader_election.cc:288] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:38.505779 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "539cf5d4596f481cb03ab4d13a19198c" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:38.505815 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "539cf5d4596f481cb03ab4d13a19198c" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:38.505859 22845 raft_consensus.cc:3037] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.505903 22979 raft_consensus.cc:3037] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.506717 22845 raft_consensus.cc:2454] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:38.506809 22979 raft_consensus.cc:2454] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:38.507033 22653 leader_election.cc:302] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:38.507179 24280 raft_consensus.cc:2781] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:38.507339 24280 raft_consensus.cc:686] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:38.507472 24280 consensus_queue.cc:237] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.509114 22441 catalog_manager.cc:5484] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:38.617501 24472 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:38.617731 24472 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:38.620344 24472 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_15'
I20240502 07:58:38.688117 22845 raft_consensus.cc:1261] T 44f7d6f2a16d4c6b88526760cacc97a5 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:38.688311 22979 raft_consensus.cc:1261] T 44f7d6f2a16d4c6b88526760cacc97a5 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:38.688735 24184 consensus_queue.cc:1035] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:38.689013 24184 consensus_queue.cc:1035] T 44f7d6f2a16d4c6b88526760cacc97a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:38.689045 22715 raft_consensus.cc:1261] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:38.690325 24311 consensus_queue.cc:1035] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:38.693818 22848 raft_consensus.cc:1261] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:38.694159 24184 consensus_queue.cc:1035] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:38.696408 22977 raft_consensus.cc:1261] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:38.696870 22847 raft_consensus.cc:1261] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:38.697518 24280 consensus_queue.cc:1035] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:38.697679 24470 consensus_queue.cc:1035] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:38.699813 24449 mvcc.cc:205] Tried to move back new op lower bound from 7023151999743148032 to 7023151998311366656. Current Snapshot: MvccSnapshot[applied={T|T < 7023151999743148032}]
INSERT report
rows total: 2000
time total: 99.5397 ms
time per row: 0.0497699 ms
I20240502 07:58:38.760821 22441 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_16"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:38.766052 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 905422dc16e74174b4f88aa87cbee3d3 (DEFAULT_TABLE table=rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.766052 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 905422dc16e74174b4f88aa87cbee3d3 (DEFAULT_TABLE table=rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.766319 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 905422dc16e74174b4f88aa87cbee3d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.766319 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 905422dc16e74174b4f88aa87cbee3d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.768244 24301 tablet_bootstrap.cc:492] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:38.768285 24298 tablet_bootstrap.cc:492] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:38.768937 22695 tablet_service.cc:1467] Processing CreateTablet for tablet eb0a7b1308114769b556795ae18a746e (DEFAULT_TABLE table=rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.769068 24301 tablet_bootstrap.cc:654] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.769106 24298 tablet_bootstrap.cc:654] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.769191 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb0a7b1308114769b556795ae18a746e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.769795 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 905422dc16e74174b4f88aa87cbee3d3 (DEFAULT_TABLE table=rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.769984 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 905422dc16e74174b4f88aa87cbee3d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.770397 24301 tablet_bootstrap.cc:492] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:38.770458 24301 ts_tablet_manager.cc:1367] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:38.770840 24301 raft_consensus.cc:348] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:38.770925 24301 raft_consensus.cc:374] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.770951 24301 raft_consensus.cc:729] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.771049 24301 consensus_queue.cc:260] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:38.771093 24299 tablet_bootstrap.cc:492] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:38.771311 24301 ts_tablet_manager.cc:1398] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:38.771868 24299 tablet_bootstrap.cc:654] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.772244 22959 tablet_service.cc:1467] Processing CreateTablet for tablet eb0a7b1308114769b556795ae18a746e (DEFAULT_TABLE table=rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.772523 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb0a7b1308114769b556795ae18a746e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.772897 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 887c4c76980d4672aec02b2dbd4d8c89 (DEFAULT_TABLE table=rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.773108 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 887c4c76980d4672aec02b2dbd4d8c89. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.773099 22564 tablet_service.cc:1467] Processing CreateTablet for tablet eb0a7b1308114769b556795ae18a746e (DEFAULT_TABLE table=rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.773285 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb0a7b1308114769b556795ae18a746e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.773734 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 887c4c76980d4672aec02b2dbd4d8c89 (DEFAULT_TABLE table=rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.773932 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 887c4c76980d4672aec02b2dbd4d8c89. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.775034 24302 tablet_bootstrap.cc:492] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:38.775224 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 887c4c76980d4672aec02b2dbd4d8c89 (DEFAULT_TABLE table=rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:38.775369 24298 tablet_bootstrap.cc:492] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:38.775413 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 887c4c76980d4672aec02b2dbd4d8c89. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:38.775436 24298 ts_tablet_manager.cc:1367] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 07:58:38.775806 24302 tablet_bootstrap.cc:654] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.775777 24298 raft_consensus.cc:348] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:38.775846 24298 raft_consensus.cc:374] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.775863 24298 raft_consensus.cc:729] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.775966 24298 consensus_queue.cc:260] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:38.776216 24298 ts_tablet_manager.cc:1398] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:38.776348 24301 tablet_bootstrap.cc:492] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:38.777091 24301 tablet_bootstrap.cc:654] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.777964 24301 tablet_bootstrap.cc:492] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:38.778035 24301 ts_tablet_manager.cc:1367] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:38.778458 24301 raft_consensus.cc:348] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.778553 24301 raft_consensus.cc:374] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.778581 24301 raft_consensus.cc:729] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.778676 24301 consensus_queue.cc:260] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.778946 24301 ts_tablet_manager.cc:1398] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:38.779083 24301 tablet_bootstrap.cc:492] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:38.779722 24301 tablet_bootstrap.cc:654] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.780740 24298 tablet_bootstrap.cc:492] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:38.780856 24301 tablet_bootstrap.cc:492] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:38.781507 24298 tablet_bootstrap.cc:654] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.781718 24301 ts_tablet_manager.cc:1367] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.001s
I20240502 07:58:38.782099 24302 tablet_bootstrap.cc:492] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:38.782184 24302 ts_tablet_manager.cc:1367] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 07:58:38.782194 24301 raft_consensus.cc:348] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.782413 24301 raft_consensus.cc:374] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.782486 24301 raft_consensus.cc:729] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.782677 24302 raft_consensus.cc:348] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.782766 24302 raft_consensus.cc:374] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.782800 24302 raft_consensus.cc:729] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.782677 24301 consensus_queue.cc:260] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.782928 24299 tablet_bootstrap.cc:492] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:38.782912 24302 consensus_queue.cc:260] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.783028 24299 ts_tablet_manager.cc:1367] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.012s user 0.001s sys 0.000s
I20240502 07:58:38.783181 24302 ts_tablet_manager.cc:1398] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:38.783336 24301 ts_tablet_manager.cc:1398] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:38.783633 24302 tablet_bootstrap.cc:492] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:38.783726 24299 raft_consensus.cc:348] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.783813 24299 raft_consensus.cc:374] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.783840 24299 raft_consensus.cc:729] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.783975 24298 tablet_bootstrap.cc:492] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:38.783941 24299 consensus_queue.cc:260] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.784049 24298 ts_tablet_manager.cc:1367] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:38.784205 24299 ts_tablet_manager.cc:1398] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:38.784369 24299 tablet_bootstrap.cc:492] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:38.784405 24302 tablet_bootstrap.cc:654] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.785336 24299 tablet_bootstrap.cc:654] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:38.785667 24302 tablet_bootstrap.cc:492] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:38.785069 24298 raft_consensus.cc:348] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.785733 24302 ts_tablet_manager.cc:1367] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:38.785766 24298 raft_consensus.cc:374] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.785800 24298 raft_consensus.cc:729] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.785940 24298 consensus_queue.cc:260] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.786156 24302 raft_consensus.cc:348] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.786255 24302 raft_consensus.cc:374] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.786283 24302 raft_consensus.cc:729] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.786379 24302 consensus_queue.cc:260] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.786607 24302 ts_tablet_manager.cc:1398] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:38.787875 24299 tablet_bootstrap.cc:492] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:38.787953 24299 ts_tablet_manager.cc:1367] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:38.788426 24299 raft_consensus.cc:348] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:38.788523 24299 raft_consensus.cc:374] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:38.788553 24299 raft_consensus.cc:729] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:38.788697 24299 consensus_queue.cc:260] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:38.788961 24299 ts_tablet_manager.cc:1398] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:38.789001 24298 ts_tablet_manager.cc:1398] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:38.792083 24114 raft_consensus.cc:482] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:38.792167 24114 raft_consensus.cc:504] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.792541 24114 leader_election.cc:288] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:38.792877 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb0a7b1308114769b556795ae18a746e" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:38.792877 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb0a7b1308114769b556795ae18a746e" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:38.792987 22586 raft_consensus.cc:2454] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:38.792987 22715 raft_consensus.cc:2454] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:38.793227 22912 leader_election.cc:302] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:38.793375 24114 raft_consensus.cc:2781] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:38.793426 24114 raft_consensus.cc:482] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:38.793473 24114 raft_consensus.cc:3037] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.794142 24114 raft_consensus.cc:504] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.794394 24114 leader_election.cc:288] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:38.794615 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb0a7b1308114769b556795ae18a746e" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:38.794739 22715 raft_consensus.cc:3037] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.794819 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb0a7b1308114769b556795ae18a746e" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:38.794909 22586 raft_consensus.cc:3037] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.795605 22715 raft_consensus.cc:2454] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:38.795748 22586 raft_consensus.cc:2454] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:38.795909 22912 leader_election.cc:302] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:38.796074 24114 raft_consensus.cc:2781] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:38.796151 24114 raft_consensus.cc:686] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:38.796277 24114 consensus_queue.cc:237] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.797775 22441 catalog_manager.cc:5484] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:38.839635 24311 raft_consensus.cc:482] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:38.839733 24311 raft_consensus.cc:504] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.840072 24311 leader_election.cc:288] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:38.840314 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "887c4c76980d4672aec02b2dbd4d8c89" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:38.840334 22977 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "887c4c76980d4672aec02b2dbd4d8c89" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:38.840415 22848 raft_consensus.cc:2454] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:38.840430 22977 raft_consensus.cc:2454] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:38.840638 22522 leader_election.cc:302] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:38.840795 24311 raft_consensus.cc:2781] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:38.840847 24311 raft_consensus.cc:482] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:38.840870 24311 raft_consensus.cc:3037] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.841712 24311 raft_consensus.cc:504] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.841938 24311 leader_election.cc:288] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:38.842185 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "887c4c76980d4672aec02b2dbd4d8c89" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:38.842219 22977 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "887c4c76980d4672aec02b2dbd4d8c89" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:38.842269 22848 raft_consensus.cc:3037] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.842299 22977 raft_consensus.cc:3037] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:38.843003 22977 raft_consensus.cc:2454] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:38.842999 22848 raft_consensus.cc:2454] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:38.843228 22520 leader_election.cc:302] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:38.843408 24311 raft_consensus.cc:2781] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:38.843487 24311 raft_consensus.cc:686] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:38.843627 24311 consensus_queue.cc:237] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:38.845170 22441 catalog_manager.cc:5484] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:39.180250 24081 consensus_queue.cc:1035] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:39.183095 24015 consensus_queue.cc:1035] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:39.189055 24311 raft_consensus.cc:482] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:39.189129 24311 raft_consensus.cc:504] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:39.189401 24311 leader_election.cc:288] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:39.189661 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "905422dc16e74174b4f88aa87cbee3d3" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:39.189674 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "905422dc16e74174b4f88aa87cbee3d3" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:39.189765 22715 raft_consensus.cc:2454] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:39.189771 22848 raft_consensus.cc:2454] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:39.189972 22519 leader_election.cc:302] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:39.190132 24311 raft_consensus.cc:2781] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:39.190187 24311 raft_consensus.cc:482] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:39.190214 24311 raft_consensus.cc:3037] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:39.190927 24311 raft_consensus.cc:504] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:39.191222 24311 leader_election.cc:288] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:39.191468 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "905422dc16e74174b4f88aa87cbee3d3" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:39.191483 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "905422dc16e74174b4f88aa87cbee3d3" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:39.191558 22848 raft_consensus.cc:3037] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:39.191581 22715 raft_consensus.cc:3037] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:39.192229 22848 raft_consensus.cc:2454] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:39.192292 22715 raft_consensus.cc:2454] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:39.192545 22519 leader_election.cc:302] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:39.192726 24311 raft_consensus.cc:2781] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:39.192806 24311 raft_consensus.cc:686] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:39.192926 24311 consensus_queue.cc:237] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:39.194315 22441 catalog_manager.cc:5484] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:39.325745 24525 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:39.325937 24525 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:39.328639 24525 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_16'
I20240502 07:58:39.428862 22717 raft_consensus.cc:1261] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:39.429123 22848 raft_consensus.cc:1261] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:39.430008 24456 consensus_queue.cc:1035] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:39.430493 24311 consensus_queue.cc:1035] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:39.433027 22848 raft_consensus.cc:1261] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:39.433893 24456 consensus_queue.cc:1035] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:39.434074 22977 raft_consensus.cc:1261] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:39.434464 24311 consensus_queue.cc:1035] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 124.205 ms
time per row: 0.0621023 ms
I20240502 07:58:39.489773 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_17"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:39.494530 22827 tablet_service.cc:1467] Processing CreateTablet for tablet bd1c2abfe2a3416bb10d6c14d3fed46a (DEFAULT_TABLE table=rebalance_test_table_17 [id=35a66cc6bd364d818b35bceada233a98]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:39.494627 22959 tablet_service.cc:1467] Processing CreateTablet for tablet bd1c2abfe2a3416bb10d6c14d3fed46a (DEFAULT_TABLE table=rebalance_test_table_17 [id=35a66cc6bd364d818b35bceada233a98]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:39.494793 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd1c2abfe2a3416bb10d6c14d3fed46a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:39.494820 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd1c2abfe2a3416bb10d6c14d3fed46a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:39.494980 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 4363e32404304568b94f615e4e29f883 (DEFAULT_TABLE table=rebalance_test_table_17 [id=35a66cc6bd364d818b35bceada233a98]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:39.495200 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4363e32404304568b94f615e4e29f883. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:39.496994 24582 tablet_bootstrap.cc:492] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:39.496977 22697 tablet_service.cc:1467] Processing CreateTablet for tablet bd1c2abfe2a3416bb10d6c14d3fed46a (DEFAULT_TABLE table=rebalance_test_table_17 [id=35a66cc6bd364d818b35bceada233a98]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:39.497071 22828 tablet_service.cc:1467] Processing CreateTablet for tablet dd815609d7324cc588f02b6cb40cfbfd (DEFAULT_TABLE table=rebalance_test_table_17 [id=35a66cc6bd364d818b35bceada233a98]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:39.497083 22566 tablet_service.cc:1467] Processing CreateTablet for tablet dd815609d7324cc588f02b6cb40cfbfd (DEFAULT_TABLE table=rebalance_test_table_17 [id=35a66cc6bd364d818b35bceada233a98]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:39.497200 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd1c2abfe2a3416bb10d6c14d3fed46a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:39.497264 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dd815609d7324cc588f02b6cb40cfbfd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:39.497301 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dd815609d7324cc588f02b6cb40cfbfd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:39.497860 24582 tablet_bootstrap.cc:654] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:39.499369 24585 tablet_bootstrap.cc:492] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:39.499542 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 4363e32404304568b94f615e4e29f883 (DEFAULT_TABLE table=rebalance_test_table_17 [id=35a66cc6bd364d818b35bceada233a98]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:39.499650 22695 tablet_service.cc:1467] Processing CreateTablet for tablet dd815609d7324cc588f02b6cb40cfbfd (DEFAULT_TABLE table=rebalance_test_table_17 [id=35a66cc6bd364d818b35bceada233a98]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:39.499737 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4363e32404304568b94f615e4e29f883. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:39.499805 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dd815609d7324cc588f02b6cb40cfbfd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:39.500178 24585 tablet_bootstrap.cc:654] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:39.500324 24582 tablet_bootstrap.cc:492] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:39.500386 24582 ts_tablet_manager.cc:1367] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:39.500730 24586 tablet_bootstrap.cc:492] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:39.500828 24582 raft_consensus.cc:348] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.500909 24582 raft_consensus.cc:374] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:39.500936 24582 raft_consensus.cc:729] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:39.501034 24582 consensus_queue.cc:260] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.501287 24582 ts_tablet_manager.cc:1398] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:39.501703 24582 tablet_bootstrap.cc:492] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:39.501673 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 4363e32404304568b94f615e4e29f883 (DEFAULT_TABLE table=rebalance_test_table_17 [id=35a66cc6bd364d818b35bceada233a98]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:39.501855 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4363e32404304568b94f615e4e29f883. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:39.501979 24586 tablet_bootstrap.cc:654] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:39.502496 24582 tablet_bootstrap.cc:654] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:39.503116 24586 tablet_bootstrap.cc:492] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:39.503602 24582 tablet_bootstrap.cc:492] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:39.503669 24582 ts_tablet_manager.cc:1367] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:39.504036 24582 raft_consensus.cc:348] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:39.504124 24586 ts_tablet_manager.cc:1367] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:39.504132 24582 raft_consensus.cc:374] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:39.504165 24582 raft_consensus.cc:729] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:39.504292 24582 consensus_queue.cc:260] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:39.504519 24582 ts_tablet_manager.cc:1398] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:39.504616 24586 raft_consensus.cc:348] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.504791 24583 tablet_bootstrap.cc:492] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:39.504792 24586 raft_consensus.cc:374] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:39.505576 24583 tablet_bootstrap.cc:654] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:39.504834 24586 raft_consensus.cc:729] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:39.505697 24586 consensus_queue.cc:260] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.505964 24586 ts_tablet_manager.cc:1398] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:39.506668 24585 tablet_bootstrap.cc:492] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:39.506696 24583 tablet_bootstrap.cc:492] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:39.506742 24585 ts_tablet_manager.cc:1367] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 07:58:39.506759 24583 ts_tablet_manager.cc:1367] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:39.507122 24585 raft_consensus.cc:348] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:39.507220 24585 raft_consensus.cc:374] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:39.507248 24585 raft_consensus.cc:729] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:39.507393 24585 consensus_queue.cc:260] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:39.507766 24585 ts_tablet_manager.cc:1398] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:39.507786 24583 raft_consensus.cc:348] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:39.507938 24583 raft_consensus.cc:374] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:39.508001 24583 raft_consensus.cc:729] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:39.508257 24583 consensus_queue.cc:260] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:39.508415 24585 tablet_bootstrap.cc:492] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:39.509599 24585 tablet_bootstrap.cc:654] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:39.510062 24583 ts_tablet_manager.cc:1398] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:39.510756 24585 tablet_bootstrap.cc:492] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:39.510820 24583 tablet_bootstrap.cc:492] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:39.510836 24585 ts_tablet_manager.cc:1367] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:39.511283 24585 raft_consensus.cc:348] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.511377 24585 raft_consensus.cc:374] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:39.511413 24585 raft_consensus.cc:729] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:39.511515 24585 consensus_queue.cc:260] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.511662 24583 tablet_bootstrap.cc:654] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:39.511775 24585 ts_tablet_manager.cc:1398] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:39.511921 24585 tablet_bootstrap.cc:492] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:39.512785 24585 tablet_bootstrap.cc:654] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:39.513309 24583 tablet_bootstrap.cc:492] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:39.513406 24583 ts_tablet_manager.cc:1367] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:39.513805 24585 tablet_bootstrap.cc:492] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:39.513860 24583 raft_consensus.cc:348] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.513917 24585 ts_tablet_manager.cc:1367] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.002s
I20240502 07:58:39.513952 24583 raft_consensus.cc:374] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:39.513990 24583 raft_consensus.cc:729] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:39.514091 24583 consensus_queue.cc:260] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.514339 24583 ts_tablet_manager.cc:1398] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:39.514367 24585 raft_consensus.cc:348] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.514520 24583 tablet_bootstrap.cc:492] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:39.514516 24585 raft_consensus.cc:374] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:39.514559 24585 raft_consensus.cc:729] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:39.514696 24585 consensus_queue.cc:260] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.515022 24585 ts_tablet_manager.cc:1398] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:39.515275 24583 tablet_bootstrap.cc:654] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:39.516155 24583 tablet_bootstrap.cc:492] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:39.516222 24583 ts_tablet_manager.cc:1367] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:39.516530 24583 raft_consensus.cc:348] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.516640 24583 raft_consensus.cc:374] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:39.516696 24583 raft_consensus.cc:729] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:39.516819 24583 consensus_queue.cc:260] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.517129 24583 ts_tablet_manager.cc:1398] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:39.524149 24521 raft_consensus.cc:482] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:39.524214 24521 raft_consensus.cc:504] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:39.524454 24521 leader_election.cc:288] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:39.524740 22977 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd1c2abfe2a3416bb10d6c14d3fed46a" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:39.524767 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd1c2abfe2a3416bb10d6c14d3fed46a" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:39.524834 22977 raft_consensus.cc:2454] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:39.524869 22715 raft_consensus.cc:2454] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:39.525069 22782 leader_election.cc:302] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:39.525235 24521 raft_consensus.cc:2781] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:39.525285 24521 raft_consensus.cc:482] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:39.525311 24521 raft_consensus.cc:3037] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:39.525956 24521 raft_consensus.cc:504] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:39.526203 24521 leader_election.cc:288] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:39.526430 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd1c2abfe2a3416bb10d6c14d3fed46a" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:39.526466 22977 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd1c2abfe2a3416bb10d6c14d3fed46a" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:39.526513 22715 raft_consensus.cc:3037] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:39.526546 22977 raft_consensus.cc:3037] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:39.527129 22715 raft_consensus.cc:2454] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:39.527127 22977 raft_consensus.cc:2454] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:39.527391 22782 leader_election.cc:302] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:39.527556 24521 raft_consensus.cc:2781] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:39.527736 24521 raft_consensus.cc:686] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:39.528064 24521 consensus_queue.cc:237] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:39.529568 22442 catalog_manager.cc:5484] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:39.704792 24573 raft_consensus.cc:482] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:39.704898 24573 raft_consensus.cc:504] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.705348 24573 leader_election.cc:288] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:39.705636 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dd815609d7324cc588f02b6cb40cfbfd" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:39.705763 22715 raft_consensus.cc:2454] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:39.705889 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dd815609d7324cc588f02b6cb40cfbfd" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:39.706015 22845 raft_consensus.cc:2454] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:39.706008 22519 leader_election.cc:302] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:39.706151 24573 raft_consensus.cc:2781] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:39.706200 24573 raft_consensus.cc:482] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:39.706245 24573 raft_consensus.cc:3037] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:39.706961 24573 raft_consensus.cc:504] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.707228 24573 leader_election.cc:288] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:39.707459 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dd815609d7324cc588f02b6cb40cfbfd" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:39.707494 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "dd815609d7324cc588f02b6cb40cfbfd" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:39.707549 22715 raft_consensus.cc:3037] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:39.707583 22845 raft_consensus.cc:3037] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:39.708206 22715 raft_consensus.cc:2454] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:39.708247 22845 raft_consensus.cc:2454] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:39.708453 22522 leader_election.cc:302] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:39.708693 24573 raft_consensus.cc:2781] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:39.708776 24573 raft_consensus.cc:686] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:39.708899 24573 consensus_queue.cc:237] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.710827 22442 catalog_manager.cc:5484] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:39.785722 24521 raft_consensus.cc:482] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:39.785828 24521 raft_consensus.cc:504] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.786202 24521 leader_election.cc:288] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:39.786520 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4363e32404304568b94f615e4e29f883" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:39.786566 22977 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4363e32404304568b94f615e4e29f883" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:39.786672 22977 raft_consensus.cc:2454] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:39.786672 22715 raft_consensus.cc:2454] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:39.786897 22781 leader_election.cc:302] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:39.787055 24521 raft_consensus.cc:2781] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:39.787128 24521 raft_consensus.cc:482] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:39.787154 24521 raft_consensus.cc:3037] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:39.788232 24521 raft_consensus.cc:504] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.788530 24521 leader_election.cc:288] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:39.788802 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4363e32404304568b94f615e4e29f883" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:39.788825 22977 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4363e32404304568b94f615e4e29f883" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:39.788893 22715 raft_consensus.cc:3037] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:39.788923 22977 raft_consensus.cc:3037] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:39.789633 22715 raft_consensus.cc:2454] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:39.789659 22977 raft_consensus.cc:2454] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:39.789861 22781 leader_election.cc:302] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:39.790021 24521 raft_consensus.cc:2781] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:39.790107 24521 raft_consensus.cc:686] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:39.790231 24521 consensus_queue.cc:237] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:39.792160 22442 catalog_manager.cc:5484] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } }
W20240502 07:58:39.965339 24599 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:39.965603 24599 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:39.968238 24599 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_17'
I20240502 07:58:40.007386 24595 consensus_queue.cc:1035] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:40.021378 24595 consensus_queue.cc:1035] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:40.058475 22848 raft_consensus.cc:1261] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:40.058888 24573 consensus_queue.cc:1035] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:40.064124 22715 raft_consensus.cc:1261] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:40.064482 24573 consensus_queue.cc:1035] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:40.070300 22979 raft_consensus.cc:1261] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:40.070924 24637 consensus_queue.cc:1035] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:40.071532 22717 raft_consensus.cc:1261] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:40.072952 24637 consensus_queue.cc:1035] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 108.641 ms
time per row: 0.0543206 ms
I20240502 07:58:40.116829 22438 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_18"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:40.121560 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 68aff050f5f7483ca821f1d28b04c91c (DEFAULT_TABLE table=rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.121641 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 68aff050f5f7483ca821f1d28b04c91c (DEFAULT_TABLE table=rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.121838 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 68aff050f5f7483ca821f1d28b04c91c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.121834 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 68aff050f5f7483ca821f1d28b04c91c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.122298 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 45499d82e94c47b8b398c159df9fd49b (DEFAULT_TABLE table=rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.122499 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 45499d82e94c47b8b398c159df9fd49b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.124032 24655 tablet_bootstrap.cc:492] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:40.124044 22958 tablet_service.cc:1467] Processing CreateTablet for tablet b583c8775a874ad1a89c82c00c05bbbd (DEFAULT_TABLE table=rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.124215 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b583c8775a874ad1a89c82c00c05bbbd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.124373 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 68aff050f5f7483ca821f1d28b04c91c (DEFAULT_TABLE table=rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.124722 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 68aff050f5f7483ca821f1d28b04c91c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.124868 24655 tablet_bootstrap.cc:654] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.124373 22695 tablet_service.cc:1467] Processing CreateTablet for tablet b583c8775a874ad1a89c82c00c05bbbd (DEFAULT_TABLE table=rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.125245 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b583c8775a874ad1a89c82c00c05bbbd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.125485 24654 tablet_bootstrap.cc:492] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:40.126294 24654 tablet_bootstrap.cc:654] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.127121 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 45499d82e94c47b8b398c159df9fd49b (DEFAULT_TABLE table=rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.127380 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 45499d82e94c47b8b398c159df9fd49b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.127583 24654 tablet_bootstrap.cc:492] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:40.127686 24654 ts_tablet_manager.cc:1367] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:40.128018 24654 raft_consensus.cc:348] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.128103 24654 raft_consensus.cc:374] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.128134 24654 raft_consensus.cc:729] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.128540 22564 tablet_service.cc:1467] Processing CreateTablet for tablet b583c8775a874ad1a89c82c00c05bbbd (DEFAULT_TABLE table=rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.128772 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b583c8775a874ad1a89c82c00c05bbbd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.129101 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 45499d82e94c47b8b398c159df9fd49b (DEFAULT_TABLE table=rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.129307 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 45499d82e94c47b8b398c159df9fd49b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.129793 24657 tablet_bootstrap.cc:492] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:40.130765 24657 tablet_bootstrap.cc:654] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.128255 24654 consensus_queue.cc:260] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.131177 24654 ts_tablet_manager.cc:1398] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:40.131345 24654 tablet_bootstrap.cc:492] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:40.131546 24659 tablet_bootstrap.cc:492] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:40.132278 24654 tablet_bootstrap.cc:654] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.132530 24659 tablet_bootstrap.cc:654] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.132548 24657 tablet_bootstrap.cc:492] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:40.132627 24657 ts_tablet_manager.cc:1367] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:40.133143 24657 raft_consensus.cc:348] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.133250 24657 raft_consensus.cc:374] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.133281 24657 raft_consensus.cc:729] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.133404 24657 consensus_queue.cc:260] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.133735 24657 ts_tablet_manager.cc:1398] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:40.133869 24657 tablet_bootstrap.cc:492] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:40.133989 24659 tablet_bootstrap.cc:492] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:40.134047 24659 ts_tablet_manager.cc:1367] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:40.134111 24655 tablet_bootstrap.cc:492] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:40.134172 24655 ts_tablet_manager.cc:1367] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.010s user 0.001s sys 0.000s
I20240502 07:58:40.134431 24655 raft_consensus.cc:348] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.134423 24659 raft_consensus.cc:348] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:40.134492 24655 raft_consensus.cc:374] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.134510 24655 raft_consensus.cc:729] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.134518 24659 raft_consensus.cc:374] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.134554 24659 raft_consensus.cc:729] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.134563 24657 tablet_bootstrap.cc:654] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.134599 24655 consensus_queue.cc:260] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.134668 24659 consensus_queue.cc:260] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:40.134819 24655 ts_tablet_manager.cc:1398] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:40.135025 24659 ts_tablet_manager.cc:1398] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:40.135443 24655 tablet_bootstrap.cc:492] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:40.136992 24657 tablet_bootstrap.cc:492] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:40.137041 24655 tablet_bootstrap.cc:654] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.137069 24657 ts_tablet_manager.cc:1367] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:40.137445 24657 raft_consensus.cc:348] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.137521 24657 raft_consensus.cc:374] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.137549 24657 raft_consensus.cc:729] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.137679 24657 consensus_queue.cc:260] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.137897 24657 ts_tablet_manager.cc:1398] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:40.138545 24655 tablet_bootstrap.cc:492] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:40.138620 24655 ts_tablet_manager.cc:1367] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:40.139190 24655 raft_consensus.cc:348] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.139287 24655 raft_consensus.cc:374] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.139340 24655 raft_consensus.cc:729] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.139444 24655 consensus_queue.cc:260] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.139756 24655 ts_tablet_manager.cc:1398] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:40.139899 24655 tablet_bootstrap.cc:492] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:40.140641 24655 tablet_bootstrap.cc:654] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.141393 24654 tablet_bootstrap.cc:492] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:40.141469 24654 ts_tablet_manager.cc:1367] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.000s
I20240502 07:58:40.141829 24654 raft_consensus.cc:348] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:40.141939 24654 raft_consensus.cc:374] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.141971 24654 raft_consensus.cc:729] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.142078 24654 consensus_queue.cc:260] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:40.142362 24654 ts_tablet_manager.cc:1398] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:40.142520 24654 tablet_bootstrap.cc:492] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:40.143290 24654 tablet_bootstrap.cc:654] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.144161 24655 tablet_bootstrap.cc:492] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:40.144230 24655 ts_tablet_manager.cc:1367] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.001s
I20240502 07:58:40.144519 24655 raft_consensus.cc:348] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:40.144589 24655 raft_consensus.cc:374] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.144629 24655 raft_consensus.cc:729] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.144758 24654 tablet_bootstrap.cc:492] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:40.144760 24655 consensus_queue.cc:260] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:40.144834 24654 ts_tablet_manager.cc:1367] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:40.144984 24655 ts_tablet_manager.cc:1398] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:40.145769 24654 raft_consensus.cc:348] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.145920 24654 raft_consensus.cc:374] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.145961 24654 raft_consensus.cc:729] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.146083 24654 consensus_queue.cc:260] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.146358 24654 ts_tablet_manager.cc:1398] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:40.152734 24456 raft_consensus.cc:482] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:40.152808 24456 raft_consensus.cc:504] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.153148 24456 leader_election.cc:288] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:40.153445 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "68aff050f5f7483ca821f1d28b04c91c" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:40.153549 22717 raft_consensus.cc:2454] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:40.153803 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "68aff050f5f7483ca821f1d28b04c91c" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:40.153848 22519 leader_election.cc:302] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:40.153903 22979 raft_consensus.cc:2454] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:40.153981 24573 raft_consensus.cc:2781] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:40.154021 24573 raft_consensus.cc:482] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:40.154052 24573 raft_consensus.cc:3037] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.154742 24573 raft_consensus.cc:504] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.154991 24573 leader_election.cc:288] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:40.155210 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "68aff050f5f7483ca821f1d28b04c91c" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:40.155299 22979 raft_consensus.cc:3037] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.155315 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "68aff050f5f7483ca821f1d28b04c91c" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:40.155411 22717 raft_consensus.cc:3037] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.156122 22979 raft_consensus.cc:2454] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:40.156195 22717 raft_consensus.cc:2454] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:40.156404 22520 leader_election.cc:302] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:40.156563 24573 raft_consensus.cc:2781] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:40.156713 24573 raft_consensus.cc:686] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:40.156852 24573 consensus_queue.cc:237] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.158684 22439 catalog_manager.cc:5484] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:40.212285 24637 raft_consensus.cc:482] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:40.212386 24637 raft_consensus.cc:504] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:40.212832 24637 leader_election.cc:288] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:40.213124 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "45499d82e94c47b8b398c159df9fd49b" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:40.213135 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "45499d82e94c47b8b398c159df9fd49b" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:40.213238 22979 raft_consensus.cc:2454] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:40.213248 22584 raft_consensus.cc:2454] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:40.213474 22782 leader_election.cc:302] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:40.213624 24637 raft_consensus.cc:2781] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:40.213685 24637 raft_consensus.cc:482] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:40.213733 24637 raft_consensus.cc:3037] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.214475 24637 raft_consensus.cc:504] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:40.214722 24637 leader_election.cc:288] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:40.215018 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "45499d82e94c47b8b398c159df9fd49b" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:40.215024 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "45499d82e94c47b8b398c159df9fd49b" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:40.215102 22979 raft_consensus.cc:3037] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.215112 22584 raft_consensus.cc:3037] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.215778 22584 raft_consensus.cc:2454] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:40.215781 22979 raft_consensus.cc:2454] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:40.216056 22784 leader_election.cc:302] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:40.216306 24637 raft_consensus.cc:2781] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:40.216379 24637 raft_consensus.cc:686] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:40.216503 24637 consensus_queue.cc:237] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:40.218340 22439 catalog_manager.cc:5484] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:40.393071 24456 raft_consensus.cc:482] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:40.393174 24456 raft_consensus.cc:504] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.393561 24456 leader_election.cc:288] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:40.393921 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b583c8775a874ad1a89c82c00c05bbbd" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:40.393891 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b583c8775a874ad1a89c82c00c05bbbd" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:40.394038 22717 raft_consensus.cc:2454] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:40.394042 22979 raft_consensus.cc:2454] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:40.394294 22519 leader_election.cc:302] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:40.394446 24456 raft_consensus.cc:2781] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:40.394505 24456 raft_consensus.cc:482] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:40.394528 24456 raft_consensus.cc:3037] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.395247 24456 raft_consensus.cc:504] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.395511 24456 leader_election.cc:288] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:40.395779 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b583c8775a874ad1a89c82c00c05bbbd" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:40.395787 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b583c8775a874ad1a89c82c00c05bbbd" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:40.395880 22979 raft_consensus.cc:3037] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.395880 22717 raft_consensus.cc:3037] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.396806 22979 raft_consensus.cc:2454] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:40.396806 22717 raft_consensus.cc:2454] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:40.397043 22519 leader_election.cc:302] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:40.397186 24456 raft_consensus.cc:2781] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:40.397264 24456 raft_consensus.cc:686] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:40.397428 24456 consensus_queue.cc:237] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.399333 22439 catalog_manager.cc:5484] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:40.525975 24670 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:40.526240 24670 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:40.529068 24670 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_18'
I20240502 07:58:40.596745 22584 raft_consensus.cc:1261] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:40.596815 22979 raft_consensus.cc:1261] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:40.597258 24637 consensus_queue.cc:1035] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:40.597676 24637 consensus_queue.cc:1035] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:40.619035 24456 consensus_queue.cc:1035] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:40.620756 22715 raft_consensus.cc:1261] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:40.621568 24573 consensus_queue.cc:1035] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:40.621990 24456 consensus_queue.cc:1035] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:40.627154 22979 raft_consensus.cc:1261] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:40.630035 24456 consensus_queue.cc:1035] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 90.7843 ms
time per row: 0.0453921 ms
I20240502 07:58:40.659693 22439 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalance_test_table_19"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:40.665364 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 023958ce03b84b2da063c13e4284e04d (DEFAULT_TABLE table=rebalance_test_table_19 [id=65deae5d69b44af784a28c39cd113241]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.665365 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 023958ce03b84b2da063c13e4284e04d (DEFAULT_TABLE table=rebalance_test_table_19 [id=65deae5d69b44af784a28c39cd113241]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.665635 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 023958ce03b84b2da063c13e4284e04d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.665637 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 023958ce03b84b2da063c13e4284e04d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.665911 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 023958ce03b84b2da063c13e4284e04d (DEFAULT_TABLE table=rebalance_test_table_19 [id=65deae5d69b44af784a28c39cd113241]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.665678 22958 tablet_service.cc:1467] Processing CreateTablet for tablet fcbab4ec41a74ff8b0c564ac7a0b044e (DEFAULT_TABLE table=rebalance_test_table_19 [id=65deae5d69b44af784a28c39cd113241]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.666121 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 023958ce03b84b2da063c13e4284e04d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.666237 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fcbab4ec41a74ff8b0c564ac7a0b044e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.667850 22959 tablet_service.cc:1467] Processing CreateTablet for tablet a8dcc36740be4ca3ac1d43591f7549bd (DEFAULT_TABLE table=rebalance_test_table_19 [id=65deae5d69b44af784a28c39cd113241]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.668058 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a8dcc36740be4ca3ac1d43591f7549bd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.668793 24722 tablet_bootstrap.cc:492] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:40.669358 22695 tablet_service.cc:1467] Processing CreateTablet for tablet a8dcc36740be4ca3ac1d43591f7549bd (DEFAULT_TABLE table=rebalance_test_table_19 [id=65deae5d69b44af784a28c39cd113241]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.669561 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a8dcc36740be4ca3ac1d43591f7549bd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.669618 24722 tablet_bootstrap.cc:654] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.669831 24724 tablet_bootstrap.cc:492] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:40.669867 22828 tablet_service.cc:1467] Processing CreateTablet for tablet a8dcc36740be4ca3ac1d43591f7549bd (DEFAULT_TABLE table=rebalance_test_table_19 [id=65deae5d69b44af784a28c39cd113241]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.670046 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a8dcc36740be4ca3ac1d43591f7549bd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.670130 22827 tablet_service.cc:1467] Processing CreateTablet for tablet fcbab4ec41a74ff8b0c564ac7a0b044e (DEFAULT_TABLE table=rebalance_test_table_19 [id=65deae5d69b44af784a28c39cd113241]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.670361 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fcbab4ec41a74ff8b0c564ac7a0b044e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.670564 24724 tablet_bootstrap.cc:654] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.671931 22566 tablet_service.cc:1467] Processing CreateTablet for tablet fcbab4ec41a74ff8b0c564ac7a0b044e (DEFAULT_TABLE table=rebalance_test_table_19 [id=65deae5d69b44af784a28c39cd113241]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:40.672129 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fcbab4ec41a74ff8b0c564ac7a0b044e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:40.672271 24723 tablet_bootstrap.cc:492] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:40.673170 24723 tablet_bootstrap.cc:654] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.673200 24725 tablet_bootstrap.cc:492] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:40.674064 24725 tablet_bootstrap.cc:654] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.674209 24723 tablet_bootstrap.cc:492] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:40.674285 24723 ts_tablet_manager.cc:1367] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:40.674592 24723 raft_consensus.cc:348] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:40.674705 24723 raft_consensus.cc:374] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.674744 24723 raft_consensus.cc:729] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.674858 24723 consensus_queue.cc:260] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:40.675022 24725 tablet_bootstrap.cc:492] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:40.675082 24725 ts_tablet_manager.cc:1367] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:40.675105 24723 ts_tablet_manager.cc:1398] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:40.675375 24723 tablet_bootstrap.cc:492] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:40.675436 24725 raft_consensus.cc:348] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.675509 24725 raft_consensus.cc:374] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.675535 24725 raft_consensus.cc:729] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.675640 24725 consensus_queue.cc:260] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.675868 24725 ts_tablet_manager.cc:1398] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:40.676025 24725 tablet_bootstrap.cc:492] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:40.676052 24723 tablet_bootstrap.cc:654] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.676849 24725 tablet_bootstrap.cc:654] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.677753 24725 tablet_bootstrap.cc:492] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:40.677829 24725 ts_tablet_manager.cc:1367] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:40.678040 24723 tablet_bootstrap.cc:492] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:40.678100 24723 ts_tablet_manager.cc:1367] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:40.678195 24725 raft_consensus.cc:348] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:40.678268 24725 raft_consensus.cc:374] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.678294 24725 raft_consensus.cc:729] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.678388 24725 consensus_queue.cc:260] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:40.678478 24723 raft_consensus.cc:348] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.678547 24723 raft_consensus.cc:374] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.678578 24723 raft_consensus.cc:729] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.678659 24725 ts_tablet_manager.cc:1398] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:40.678674 24723 consensus_queue.cc:260] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.679538 24723 ts_tablet_manager.cc:1398] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:40.679688 24723 tablet_bootstrap.cc:492] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:40.680913 24724 tablet_bootstrap.cc:492] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:40.680981 24722 tablet_bootstrap.cc:492] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:40.680994 24724 ts_tablet_manager.cc:1367] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.011s user 0.001s sys 0.000s
I20240502 07:58:40.681053 24722 ts_tablet_manager.cc:1367] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.012s user 0.001s sys 0.000s
I20240502 07:58:40.681236 24723 tablet_bootstrap.cc:654] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.681398 24724 raft_consensus.cc:348] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:40.681414 24722 raft_consensus.cc:348] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:40.681531 24722 raft_consensus.cc:374] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.681531 24724 raft_consensus.cc:374] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.681568 24722 raft_consensus.cc:729] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.681578 24724 raft_consensus.cc:729] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.681681 24722 consensus_queue.cc:260] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:40.681691 24724 consensus_queue.cc:260] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:40.681926 24724 ts_tablet_manager.cc:1398] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:40.681924 24722 ts_tablet_manager.cc:1398] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:40.682052 24724 tablet_bootstrap.cc:492] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:40.682052 24722 tablet_bootstrap.cc:492] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:40.682236 24723 tablet_bootstrap.cc:492] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:40.682298 24723 ts_tablet_manager.cc:1367] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:40.682686 24723 raft_consensus.cc:348] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:40.682775 24723 raft_consensus.cc:374] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.682806 24723 raft_consensus.cc:729] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.682842 24722 tablet_bootstrap.cc:654] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.682909 24723 consensus_queue.cc:260] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:40.683158 24723 ts_tablet_manager.cc:1398] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:40.684075 24724 tablet_bootstrap.cc:654] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:40.685060 24722 tablet_bootstrap.cc:492] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:40.685129 24722 ts_tablet_manager.cc:1367] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:40.685266 24724 tablet_bootstrap.cc:492] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:40.685322 24724 ts_tablet_manager.cc:1367] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.001s
I20240502 07:58:40.685489 24722 raft_consensus.cc:348] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.685686 24722 raft_consensus.cc:374] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.685722 24722 raft_consensus.cc:729] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.685698 24724 raft_consensus.cc:348] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:40.685770 24724 raft_consensus.cc:374] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:40.685796 24724 raft_consensus.cc:729] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:40.685828 24722 consensus_queue.cc:260] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.685889 24724 consensus_queue.cc:260] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:40.686107 24722 ts_tablet_manager.cc:1398] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:40.686105 24724 ts_tablet_manager.cc:1398] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:40.703837 24637 raft_consensus.cc:482] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:40.703917 24637 raft_consensus.cc:504] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:40.704268 24637 leader_election.cc:288] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:40.704500 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fcbab4ec41a74ff8b0c564ac7a0b044e" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:40.704582 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fcbab4ec41a74ff8b0c564ac7a0b044e" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:40.704623 22584 raft_consensus.cc:2454] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:40.704746 22978 raft_consensus.cc:2454] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:40.704941 22784 leader_election.cc:302] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:40.705113 24637 raft_consensus.cc:2781] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:40.705159 24637 raft_consensus.cc:482] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:40.705202 24637 raft_consensus.cc:3037] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.705976 24637 raft_consensus.cc:504] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:40.706250 24637 leader_election.cc:288] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:40.706506 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fcbab4ec41a74ff8b0c564ac7a0b044e" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:40.706554 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fcbab4ec41a74ff8b0c564ac7a0b044e" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:40.706633 22978 raft_consensus.cc:3037] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.706625 22584 raft_consensus.cc:3037] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.707535 22584 raft_consensus.cc:2454] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:40.707535 22978 raft_consensus.cc:2454] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:40.707792 22784 leader_election.cc:302] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:40.707914 24637 raft_consensus.cc:2781] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:40.707988 24637 raft_consensus.cc:686] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:40.708107 24637 consensus_queue.cc:237] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:40.709973 22439 catalog_manager.cc:5484] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:40.732899 24081 raft_consensus.cc:482] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:40.733000 24081 raft_consensus.cc:504] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.733310 24081 leader_election.cc:288] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:40.733587 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a8dcc36740be4ca3ac1d43591f7549bd" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:40.733624 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a8dcc36740be4ca3ac1d43591f7549bd" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:40.733729 22848 raft_consensus.cc:2454] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:40.733729 22717 raft_consensus.cc:2454] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:40.733999 22912 leader_election.cc:302] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:40.734206 24081 raft_consensus.cc:2781] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:40.734258 24081 raft_consensus.cc:482] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:40.734292 24081 raft_consensus.cc:3037] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.735100 24081 raft_consensus.cc:504] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.735397 24081 leader_election.cc:288] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:40.735674 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a8dcc36740be4ca3ac1d43591f7549bd" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:40.735680 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a8dcc36740be4ca3ac1d43591f7549bd" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:40.735778 22848 raft_consensus.cc:3037] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.735798 22717 raft_consensus.cc:3037] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.736730 22717 raft_consensus.cc:2454] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:40.736733 22848 raft_consensus.cc:2454] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:40.737003 22912 leader_election.cc:302] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:40.737147 24081 raft_consensus.cc:2781] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:40.737229 24081 raft_consensus.cc:686] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:40.737336 24081 consensus_queue.cc:237] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:40.739171 22439 catalog_manager.cc:5484] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:40.900178 24712 raft_consensus.cc:482] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:40.900285 24712 raft_consensus.cc:504] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:40.900705 24712 leader_election.cc:288] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:40.901126 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "023958ce03b84b2da063c13e4284e04d" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:40.901244 22848 raft_consensus.cc:2454] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:40.901536 22653 leader_election.cc:302] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:40.901579 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "023958ce03b84b2da063c13e4284e04d" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:40.901688 24712 raft_consensus.cc:2781] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:40.901700 22584 raft_consensus.cc:2454] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:40.901741 24712 raft_consensus.cc:482] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:40.901789 24712 raft_consensus.cc:3037] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.902748 24712 raft_consensus.cc:504] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:40.903034 24712 leader_election.cc:288] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:40.903292 22848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "023958ce03b84b2da063c13e4284e04d" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:40.903349 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "023958ce03b84b2da063c13e4284e04d" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:40.903394 22848 raft_consensus.cc:3037] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.903470 22584 raft_consensus.cc:3037] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:40.904165 22584 raft_consensus.cc:2454] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:40.904255 22848 raft_consensus.cc:2454] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:40.904450 22653 leader_election.cc:302] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:40.904618 24712 raft_consensus.cc:2781] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:40.904748 24712 raft_consensus.cc:686] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:40.905037 24712 consensus_queue.cc:237] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:40.906991 22439 catalog_manager.cc:5484] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:41.054643 24738 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:41.054826 24738 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:41.057449 24738 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_19'
I20240502 07:58:41.135917 22584 raft_consensus.cc:1261] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:41.136133 22715 raft_consensus.cc:1261] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:41.136126 22979 raft_consensus.cc:1261] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:41.136447 22848 raft_consensus.cc:1261] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:41.136566 24637 consensus_queue.cc:1035] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:41.136905 22584 raft_consensus.cc:1261] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:41.137089 24015 consensus_queue.cc:1035] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:41.137202 24081 consensus_queue.cc:1035] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:41.137388 22848 raft_consensus.cc:1261] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:41.140796 24627 consensus_queue.cc:1035] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:41.141182 24712 consensus_queue.cc:1035] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:41.147830 24700 consensus_queue.cc:1035] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 105.152 ms
time per row: 0.052576 ms
I20240502 07:58:41.205868 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.133:35459
--local_ip_for_outbound_sockets=127.16.85.133
--tserver_master_addrs=127.16.85.190:39083
--webserver_port=36449
--webserver_interface=127.16.85.133
--builtin_ntp_servers=127.16.85.148:41971
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 07:58:41.306433 24790 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:41.306627 24790 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:41.306681 24790 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:41.309657 24790 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:41.309711 24790 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:41.309794 24790 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.133
I20240502 07:58:41.313110 24790 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:41971
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.133:35459
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.16.85.133
--webserver_port=36449
--tserver_master_addrs=127.16.85.190:39083
--never_fsync=true
--heap_profile_path=/tmp/kudu.24790
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.133
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:41.313858 24790 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:41.314824 24790 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:41.322304 24796 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:41.323052 24795 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:41.323822 24798 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:41.324450 24790 server_base.cc:1034] running on GCE node
I20240502 07:58:41.324807 24790 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:41.325268 24790 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:41.326406 24790 hybrid_clock.cc:648] HybridClock initialized: now 1714636721326397 us; error 24 us; skew 500 ppm
I20240502 07:58:41.327989 24790 webserver.cc:460] Webserver started at http://127.16.85.133:36449/ using document root <none> and password file <none>
I20240502 07:58:41.328521 24790 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:41.328589 24790 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:41.331962 24790 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:41.334025 24804 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:41.334954 24790 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:41.335034 24790 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "6d02017318a2461796d431962fe6a537"
format_stamp: "Formatted at 2024-05-02 07:58:27 on dist-test-slave-7df7"
I20240502 07:58:41.335367 24790 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:41.350358 24790 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:41.350876 24790 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:41.351007 24790 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:41.351456 24790 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:41.352198 24790 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:41.352241 24790 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:41.352294 24790 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:41.352319 24790 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:41.361063 24790 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.133:35459
I20240502 07:58:41.361095 24917 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.133:35459 every 8 connection(s)
I20240502 07:58:41.362035 24790 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 07:58:41.369623 24918 heartbeater.cc:347] Connected to a master server at 127.16.85.190:39083
I20240502 07:58:41.369787 24918 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:41.370188 24918 heartbeater.cc:511] Master 127.16.85.190:39083 requested a full tablet report, sending...
I20240502 07:58:41.370918 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 24790
I20240502 07:58:41.371198 22436 ts_manager.cc:194] Re-registered known tserver with Master: 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:41.371366 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.134:34857
--local_ip_for_outbound_sockets=127.16.85.134
--tserver_master_addrs=127.16.85.190:39083
--webserver_port=46069
--webserver_interface=127.16.85.134
--builtin_ntp_servers=127.16.85.148:41971
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:41.371949 22436 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.133:47859
W20240502 07:58:41.473459 24921 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:41.473713 24921 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:41.473784 24921 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:41.476712 24921 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:41.476795 24921 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:41.476893 24921 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.134
I20240502 07:58:41.480443 24921 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:41971
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.134:34857
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.16.85.134
--webserver_port=46069
--tserver_master_addrs=127.16.85.190:39083
--never_fsync=true
--heap_profile_path=/tmp/kudu.24921
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.134
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:41.481204 24921 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:41.482376 24921 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:41.489945 24929 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:41.490795 24926 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:41.491443 24927 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:41.492058 24921 server_base.cc:1034] running on GCE node
I20240502 07:58:41.492389 24921 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:41.492985 24921 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:41.494133 24921 hybrid_clock.cc:648] HybridClock initialized: now 1714636721494118 us; error 32 us; skew 500 ppm
I20240502 07:58:41.495793 24921 webserver.cc:460] Webserver started at http://127.16.85.134:46069/ using document root <none> and password file <none>
I20240502 07:58:41.496366 24921 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:41.496454 24921 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:41.499853 24921 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:41.501602 24935 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:41.502461 24921 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:41.502574 24921 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "74f9aa91c2c448f980b512fe2e8d672e"
format_stamp: "Formatted at 2024-05-02 07:58:27 on dist-test-slave-7df7"
I20240502 07:58:41.502948 24921 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:41.522445 24921 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:41.523069 24921 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:41.523226 24921 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:41.523705 24921 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:41.524513 24921 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:41.524564 24921 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:41.524616 24921 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:41.524645 24921 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:41.532603 24921 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.134:34857
I20240502 07:58:41.532688 25049 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.134:34857 every 8 connection(s)
I20240502 07:58:41.533775 24921 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 07:58:41.536832 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 24921
I20240502 07:58:41.537163 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.135:41205
--local_ip_for_outbound_sockets=127.16.85.135
--tserver_master_addrs=127.16.85.190:39083
--webserver_port=36171
--webserver_interface=127.16.85.135
--builtin_ntp_servers=127.16.85.148:41971
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:41.541182 25050 heartbeater.cc:347] Connected to a master server at 127.16.85.190:39083
I20240502 07:58:41.541344 25050 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:41.541780 25050 heartbeater.cc:511] Master 127.16.85.190:39083 requested a full tablet report, sending...
I20240502 07:58:41.542678 22436 ts_manager.cc:194] Re-registered known tserver with Master: 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:41.543183 22436 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.134:59549
W20240502 07:58:41.641602 25053 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:41.641772 25053 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:41.641813 25053 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:41.644685 25053 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:41.644762 25053 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:41.644845 25053 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.135
I20240502 07:58:41.648149 25053 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:41971
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.135:41205
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.16.85.135
--webserver_port=36171
--tserver_master_addrs=127.16.85.190:39083
--never_fsync=true
--heap_profile_path=/tmp/kudu.25053
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.135
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:41.648928 25053 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:41.649871 25053 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:41.658083 25059 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:41.658622 25061 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:41.659229 25058 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:41.660372 25053 server_base.cc:1034] running on GCE node
I20240502 07:58:41.660710 25053 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:41.661182 25053 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:41.662328 25053 hybrid_clock.cc:648] HybridClock initialized: now 1714636721662308 us; error 40 us; skew 500 ppm
I20240502 07:58:41.664175 25053 webserver.cc:460] Webserver started at http://127.16.85.135:36171/ using document root <none> and password file <none>
I20240502 07:58:41.664865 25053 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:41.664949 25053 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:41.669184 25053 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.003s sys 0.000s
I20240502 07:58:41.671525 25067 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:41.672757 25053 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:41.672854 25053 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "4654231dfe994c1a9d2e37d9c3d75699"
format_stamp: "Formatted at 2024-05-02 07:58:27 on dist-test-slave-7df7"
I20240502 07:58:41.673326 25053 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:41.697657 25053 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:41.698192 25053 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:41.698340 25053 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:41.699012 25053 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:41.699913 25053 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:41.699973 25053 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:41.700023 25053 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:41.700052 25053 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:41.708811 25053 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.135:41205
I20240502 07:58:41.708847 25180 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.135:41205 every 8 connection(s)
I20240502 07:58:41.709719 25053 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 07:58:41.712064 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 25053
I20240502 07:58:41.712548 16726 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0LLEln/build/debug/bin/kudu
/tmp/dist-test-task0LLEln/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.16.85.136:34769
--local_ip_for_outbound_sockets=127.16.85.136
--tserver_master_addrs=127.16.85.190:39083
--webserver_port=36521
--webserver_interface=127.16.85.136
--builtin_ntp_servers=127.16.85.148:41971
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 07:58:41.719235 25181 heartbeater.cc:347] Connected to a master server at 127.16.85.190:39083
I20240502 07:58:41.719455 25181 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:41.720067 25181 heartbeater.cc:511] Master 127.16.85.190:39083 requested a full tablet report, sending...
I20240502 07:58:41.721154 22436 ts_manager.cc:194] Re-registered known tserver with Master: 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:41.721907 22436 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.135:47425
W20240502 07:58:41.817710 25184 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 07:58:41.817889 25184 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 07:58:41.817931 25184 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 07:58:41.820712 25184 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 07:58:41.820761 25184 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 07:58:41.820822 25184 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.85.136
I20240502 07:58:41.824396 25184 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.16.85.148:41971
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.16.85.136:34769
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.16.85.136
--webserver_port=36521
--tserver_master_addrs=127.16.85.190:39083
--never_fsync=true
--heap_profile_path=/tmp/kudu.25184
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.16.85.136
--log_dir=/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type DEBUG
built by None at 02 May 2024 07:43:15 UTC on 1a15035a6830
build id 85
I20240502 07:58:41.825160 25184 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 07:58:41.826208 25184 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 07:58:41.834694 25189 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:41.835491 25192 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 07:58:41.836136 25190 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 07:58:41.836839 25184 server_base.cc:1034] running on GCE node
I20240502 07:58:41.837167 25184 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 07:58:41.837643 25184 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 07:58:41.838797 25184 hybrid_clock.cc:648] HybridClock initialized: now 1714636721838774 us; error 34 us; skew 500 ppm
I20240502 07:58:41.845185 25184 webserver.cc:460] Webserver started at http://127.16.85.136:36521/ using document root <none> and password file <none>
I20240502 07:58:41.845701 25184 fs_manager.cc:359] Metadata directory not provided
I20240502 07:58:41.845762 25184 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 07:58:41.848775 25184 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.003s sys 0.000s
I20240502 07:58:41.850461 25198 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:41.851487 25184 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:41.851583 25184 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "7eced220cf1642f3873fc385b6191e69"
format_stamp: "Formatted at 2024-05-02 07:58:27 on dist-test-slave-7df7"
I20240502 07:58:41.851935 25184 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 07:58:41.878381 25184 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 07:58:41.878885 25184 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 07:58:41.879020 25184 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 07:58:41.879488 25184 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 07:58:41.880252 25184 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 07:58:41.880295 25184 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:41.880362 25184 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 07:58:41.880393 25184 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 07:58:41.888823 25184 rpc_server.cc:307] RPC server started. Bound to: 127.16.85.136:34769
I20240502 07:58:41.888875 25311 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.85.136:34769 every 8 connection(s)
I20240502 07:58:41.889694 25184 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240502 07:58:41.897732 25312 heartbeater.cc:347] Connected to a master server at 127.16.85.190:39083
I20240502 07:58:41.897919 25312 heartbeater.cc:464] Registering TS with master...
I20240502 07:58:41.898339 25312 heartbeater.cc:511] Master 127.16.85.190:39083 requested a full tablet report, sending...
I20240502 07:58:41.898577 16726 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0LLEln/build/debug/bin/kudu as pid 25184
I20240502 07:58:41.899567 22436 ts_manager.cc:194] Re-registered known tserver with Master: 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:41.900117 22436 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.85.136:57893
I20240502 07:58:41.902010 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:41.911438 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 43ff363342a547afa9edd078f1dfe6f2 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=a2b784b31a5b43b68c8b51571c83f116]), partition=
I20240502 07:58:41.911691 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 43ff363342a547afa9edd078f1dfe6f2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:41.911754 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 43ff363342a547afa9edd078f1dfe6f2 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=a2b784b31a5b43b68c8b51571c83f116]), partition=
I20240502 07:58:41.911958 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 43ff363342a547afa9edd078f1dfe6f2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:41.913983 25325 tablet_bootstrap.cc:492] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:41.915139 25325 tablet_bootstrap.cc:654] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:41.915634 25326 tablet_bootstrap.cc:492] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:41.916319 25326 tablet_bootstrap.cc:654] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:41.917249 25326 tablet_bootstrap.cc:492] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:41.917321 25326 ts_tablet_manager.cc:1367] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:41.917749 25325 tablet_bootstrap.cc:492] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:41.917804 25325 ts_tablet_manager.cc:1367] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:41.918077 25325 raft_consensus.cc:348] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:41.918360 25325 raft_consensus.cc:374] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:41.918532 25325 raft_consensus.cc:729] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:41.918560 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 43ff363342a547afa9edd078f1dfe6f2 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=a2b784b31a5b43b68c8b51571c83f116]), partition=
I20240502 07:58:41.918665 25325 consensus_queue.cc:260] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:41.919126 25325 ts_tablet_manager.cc:1398] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:41.919176 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 43ff363342a547afa9edd078f1dfe6f2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:41.919474 25326 raft_consensus.cc:348] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:41.919548 25326 raft_consensus.cc:374] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:41.919566 25326 raft_consensus.cc:729] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:41.919659 25326 consensus_queue.cc:260] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:41.921463 25326 ts_tablet_manager.cc:1398] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:41.924934 25329 tablet_bootstrap.cc:492] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:41.928210 25329 tablet_bootstrap.cc:654] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:41.928944 25329 log.cc:824] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:41.930469 25329 tablet_bootstrap.cc:492] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:41.930622 25329 ts_tablet_manager.cc:1367] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.002s
I20240502 07:58:41.933672 25329 raft_consensus.cc:348] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:41.933845 25329 raft_consensus.cc:374] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:41.933888 25329 raft_consensus.cc:729] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:41.934245 25329 consensus_queue.cc:260] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:41.934839 25312 heartbeater.cc:503] Master 127.16.85.190:39083 was elected leader, sending a full tablet report...
I20240502 07:58:41.935330 25329 ts_tablet_manager.cc:1398] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.005s user 0.004s sys 0.001s
I20240502 07:58:42.042726 25331 raft_consensus.cc:482] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:42.042878 25331 raft_consensus.cc:504] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:42.043874 25331 leader_election.cc:288] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:42.047885 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43ff363342a547afa9edd078f1dfe6f2" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:42.048017 22717 raft_consensus.cc:2454] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:42.048014 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43ff363342a547afa9edd078f1dfe6f2" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:42.048113 22845 raft_consensus.cc:2454] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:42.048349 25199 leader_election.cc:302] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:42.048570 25331 raft_consensus.cc:2781] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:42.048684 25331 raft_consensus.cc:482] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:42.048746 25331 raft_consensus.cc:3037] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:42.049712 25331 raft_consensus.cc:504] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:42.050034 25331 leader_election.cc:288] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:42.050318 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43ff363342a547afa9edd078f1dfe6f2" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:42.050360 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "43ff363342a547afa9edd078f1dfe6f2" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:42.050549 22717 raft_consensus.cc:3037] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:42.050586 22845 raft_consensus.cc:3037] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:42.051467 22845 raft_consensus.cc:2454] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:42.051501 22717 raft_consensus.cc:2454] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:42.051795 25199 leader_election.cc:302] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:42.051954 25331 raft_consensus.cc:2781] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:42.052111 25331 raft_consensus.cc:686] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:42.052412 25331 consensus_queue.cc:237] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:42.055231 22436 catalog_manager.cc:5484] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:42.140527 25313 tablet.cc:2369] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 07:58:42.148552 25115 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:42.150959 25235 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:42.152889 24984 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:42.157534 24852 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:42.162909 22435 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:42.163163 22827 tablet_service.cc:1429] Tablet server has 14 leaders and 0 scanners
I20240502 07:58:42.164618 22958 tablet_service.cc:1429] Tablet server has 20 leaders and 0 scanners
I20240502 07:58:42.172176 22695 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 07:58:42.178494 22717 raft_consensus.cc:1261] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:42.178898 25331 consensus_queue.cc:1035] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:42.180027 22845 raft_consensus.cc:1261] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:42.180553 25331 consensus_queue.cc:1035] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:42.181701 22436 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet 43ff363342a547afa9edd078f1dfe6f2 (table rebalancer_dynamic_table [id=a2b784b31a5b43b68c8b51571c83f116]). Expected version 1 got 0
I20240502 07:58:42.183344 22436 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet 43ff363342a547afa9edd078f1dfe6f2 (table rebalancer_dynamic_table [id=a2b784b31a5b43b68c8b51571c83f116]). Expected version 1 got 0
I20240502 07:58:42.191488 22566 tablet_service.cc:1429] Tablet server has 16 leaders and 0 scanners
I20240502 07:58:42.197621 22436 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet 43ff363342a547afa9edd078f1dfe6f2 (table rebalancer_dynamic_table [id=a2b784b31a5b43b68c8b51571c83f116]). Expected version 1 got 0
I20240502 07:58:42.198149 25395 mvcc.cc:205] Tried to move back new op lower bound from 7023152014037745664 to 7023152013529350144. Current Snapshot: MvccSnapshot[applied={T|T < 7023152014037745664}]
I20240502 07:58:42.198814 25398 mvcc.cc:205] Tried to move back new op lower bound from 7023152014037745664 to 7023152013529350144. Current Snapshot: MvccSnapshot[applied={T|T < 7023152014037745664}]
I20240502 07:58:42.199003 25400 tablet.cc:1722] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:42.199651 25400 tablet.cc:1620] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:42.201936 22436 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet 43ff363342a547afa9edd078f1dfe6f2 (table rebalancer_dynamic_table [id=a2b784b31a5b43b68c8b51571c83f116]). Expected version 1 got 0
I20240502 07:58:42.205266 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=a2b784b31a5b43b68c8b51571c83f116] alter complete (version 1)
I20240502 07:58:42.218163 22436 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet 43ff363342a547afa9edd078f1dfe6f2 (table rebalancer_dynamic_table [id=a2b784b31a5b43b68c8b51571c83f116]). Expected version 1 got 0
I20240502 07:58:42.218792 25401 tablet.cc:1722] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:42.219517 25401 tablet.cc:1620] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:58:42.221649 25402 tablet.cc:1722] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:42.222213 25402 tablet.cc:1620] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:58:42.312903 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:42.318181 25400 tablet.cc:1722] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:42.318477 25400 tablet.cc:1620] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:42.318756 25402 tablet.cc:1722] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:42.319015 25402 tablet.cc:1620] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:58:42.319288 25401 tablet.cc:1722] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:42.319583 25401 tablet.cc:1620] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:58:42.326282 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=a2b784b31a5b43b68c8b51571c83f116] alter complete (version 2)
I20240502 07:58:42.329615 25115 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:42.333808 22958 tablet_service.cc:1429] Tablet server has 20 leaders and 0 scanners
I20240502 07:58:42.341627 25235 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:42.341980 22566 tablet_service.cc:1429] Tablet server has 16 leaders and 0 scanners
I20240502 07:58:42.342898 22695 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 07:58:42.344249 22827 tablet_service.cc:1429] Tablet server has 14 leaders and 0 scanners
I20240502 07:58:42.351642 24984 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:42.373883 24918 heartbeater.cc:503] Master 127.16.85.190:39083 was elected leader, sending a full tablet report...
I20240502 07:58:42.377743 24852 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:42.399852 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:42.401403 22440 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_0" } modify_external_catalogs: true
I20240502 07:58:42.401520 22440 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_0" } modify_external_catalogs: true
I20240502 07:58:42.404100 22440 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 6844411c7a99471f8f5bbe36ed89f91e
I20240502 07:58:42.404456 22440 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 2dd6748f1e8a4308b89e0329c54d63ac
I20240502 07:58:42.404713 22440 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 207d85c84a6e4929b4842990860c65e6
I20240502 07:58:42.404821 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet 6844411c7a99471f8f5bbe36ed89f91e with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:42.405138 22958 tablet_service.cc:1514] Processing DeleteTablet for tablet 6844411c7a99471f8f5bbe36ed89f91e with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:42.405192 22828 tablet_service.cc:1514] Processing DeleteTablet for tablet 207d85c84a6e4929b4842990860c65e6 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:42.405192 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet 2dd6748f1e8a4308b89e0329c54d63ac with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:42.405544 25466 tablet_replica.cc:330] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:42.405553 25467 tablet_replica.cc:330] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:42.405712 22564 tablet_service.cc:1514] Processing DeleteTablet for tablet 2dd6748f1e8a4308b89e0329c54d63ac with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:42.405817 25467 raft_consensus.cc:2229] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:42.405910 25466 raft_consensus.cc:2229] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:42.405925 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 6844411c7a99471f8f5bbe36ed89f91e with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:42.405980 25467 raft_consensus.cc:2258] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.406221 25466 raft_consensus.cc:2258] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.407109 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet 2dd6748f1e8a4308b89e0329c54d63ac with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:42.407182 22958 tablet_service.cc:1514] Processing DeleteTablet for tablet 207d85c84a6e4929b4842990860c65e6 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:42.407265 25467 ts_tablet_manager.cc:1875] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.407255 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 207d85c84a6e4929b4842990860c65e6 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:42.407334 25466 ts_tablet_manager.cc:1875] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.407773 25465 tablet_replica.cc:330] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:42.409551 25467 ts_tablet_manager.cc:1888] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.409687 25467 log.cc:1196] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/207d85c84a6e4929b4842990860c65e6
I20240502 07:58:42.410038 25467 ts_tablet_manager.cc:1909] T 207d85c84a6e4929b4842990860c65e6 P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:42.410164 25466 ts_tablet_manager.cc:1888] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.410300 25466 log.cc:1196] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/6844411c7a99471f8f5bbe36ed89f91e
I20240502 07:58:42.410626 25466 ts_tablet_manager.cc:1909] T 6844411c7a99471f8f5bbe36ed89f91e P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:42.410468 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 28832d9e18ea41b194cf33d8c7971e23 (DEFAULT_TABLE table=rebalancer_extra_table_0 [id=3913f9e0fdc04fa2ab31379c9048ad9a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:42.410921 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 28832d9e18ea41b194cf33d8c7971e23. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:42.411022 25467 tablet_replica.cc:330] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:42.411139 25467 raft_consensus.cc:2229] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:42.411237 25467 raft_consensus.cc:2258] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.411671 25467 ts_tablet_manager.cc:1875] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.411703 25466 tablet_replica.cc:330] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:42.411849 25466 raft_consensus.cc:2229] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:42.412072 25466 raft_consensus.cc:2258] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.412539 25466 ts_tablet_manager.cc:1875] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.412942 25246 tablet_service.cc:1467] Processing CreateTablet for tablet e801bc67a63d4b249b12f9cbd9567582 (DEFAULT_TABLE table=rebalancer_extra_table_0 [id=3913f9e0fdc04fa2ab31379c9048ad9a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:42.413164 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e801bc67a63d4b249b12f9cbd9567582. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:42.413370 25467 ts_tablet_manager.cc:1888] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.413434 25467 log.cc:1196] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/2dd6748f1e8a4308b89e0329c54d63ac
I20240502 07:58:42.413614 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 6844411c7a99471f8f5bbe36ed89f91e (table rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]) successfully deleted
I20240502 07:58:42.413702 25467 ts_tablet_manager.cc:1909] T 2dd6748f1e8a4308b89e0329c54d63ac P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:42.414439 25466 ts_tablet_manager.cc:1888] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.414495 25466 log.cc:1196] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/207d85c84a6e4929b4842990860c65e6
I20240502 07:58:42.414734 25466 ts_tablet_manager.cc:1909] T 207d85c84a6e4929b4842990860c65e6 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:42.415356 25329 tablet_bootstrap.cc:492] T e801bc67a63d4b249b12f9cbd9567582 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:42.415305 22695 tablet_service.cc:1467] Processing CreateTablet for tablet e801bc67a63d4b249b12f9cbd9567582 (DEFAULT_TABLE table=rebalancer_extra_table_0 [id=3913f9e0fdc04fa2ab31379c9048ad9a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:42.415452 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 207d85c84a6e4929b4842990860c65e6 (table rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]) successfully deleted
I20240502 07:58:42.415575 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e801bc67a63d4b249b12f9cbd9567582. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:42.416031 25329 tablet_bootstrap.cc:654] T e801bc67a63d4b249b12f9cbd9567582 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:42.416973 25329 tablet_bootstrap.cc:492] T e801bc67a63d4b249b12f9cbd9567582 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:42.417060 25329 ts_tablet_manager.cc:1367] T e801bc67a63d4b249b12f9cbd9567582 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:42.417379 25326 tablet_bootstrap.cc:492] T e801bc67a63d4b249b12f9cbd9567582 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:42.417770 25329 raft_consensus.cc:348] T e801bc67a63d4b249b12f9cbd9567582 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:42.417867 25329 raft_consensus.cc:374] T e801bc67a63d4b249b12f9cbd9567582 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:42.417897 25329 raft_consensus.cc:729] T e801bc67a63d4b249b12f9cbd9567582 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:42.418002 25329 consensus_queue.cc:260] T e801bc67a63d4b249b12f9cbd9567582 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:42.418169 25326 tablet_bootstrap.cc:654] T e801bc67a63d4b249b12f9cbd9567582 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:42.418254 25329 ts_tablet_manager.cc:1398] T e801bc67a63d4b249b12f9cbd9567582 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:42.418409 25329 tablet_bootstrap.cc:492] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:42.408067 25465 raft_consensus.cc:2229] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:42.419996 25465 raft_consensus.cc:2258] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.420518 25329 tablet_bootstrap.cc:654] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:42.420758 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 207d85c84a6e4929b4842990860c65e6 (table rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]) successfully deleted
I20240502 07:58:42.420897 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 2dd6748f1e8a4308b89e0329c54d63ac (table rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]) successfully deleted
I20240502 07:58:42.422201 22958 tablet_service.cc:1467] Processing CreateTablet for tablet 273244b2535e4fcbbd22de0d236715c5 (DEFAULT_TABLE table=rebalancer_extra_table_0 [id=3913f9e0fdc04fa2ab31379c9048ad9a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:42.421998 24984 tablet_service.cc:1467] Processing CreateTablet for tablet 28832d9e18ea41b194cf33d8c7971e23 (DEFAULT_TABLE table=rebalancer_extra_table_0 [id=3913f9e0fdc04fa2ab31379c9048ad9a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:42.422456 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 273244b2535e4fcbbd22de0d236715c5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:42.422523 25329 tablet_bootstrap.cc:492] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:42.422582 25329 ts_tablet_manager.cc:1367] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:42.423004 24984 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 28832d9e18ea41b194cf33d8c7971e23. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:42.422977 25329 raft_consensus.cc:348] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:42.423060 25329 raft_consensus.cc:374] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:42.423089 25329 raft_consensus.cc:729] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:42.423189 25329 consensus_queue.cc:260] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:42.423274 25465 ts_tablet_manager.cc:1875] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.423449 25329 ts_tablet_manager.cc:1398] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:42.424335 25473 tablet_bootstrap.cc:492] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:42.425123 25473 tablet_bootstrap.cc:654] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:42.425328 22566 tablet_service.cc:1467] Processing CreateTablet for tablet e801bc67a63d4b249b12f9cbd9567582 (DEFAULT_TABLE table=rebalancer_extra_table_0 [id=3913f9e0fdc04fa2ab31379c9048ad9a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:42.425572 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e801bc67a63d4b249b12f9cbd9567582. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:42.427008 25473 tablet_bootstrap.cc:492] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:42.427074 25473 ts_tablet_manager.cc:1367] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:42.427392 25473 raft_consensus.cc:348] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:42.427476 25473 raft_consensus.cc:374] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:42.427500 25473 raft_consensus.cc:729] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:42.427594 25473 consensus_queue.cc:260] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:42.427840 25473 ts_tablet_manager.cc:1398] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:42.428470 25465 ts_tablet_manager.cc:1888] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.428550 25465 log.cc:1196] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/6844411c7a99471f8f5bbe36ed89f91e
I20240502 07:58:42.428344 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 273244b2535e4fcbbd22de0d236715c5 (DEFAULT_TABLE table=rebalancer_extra_table_0 [id=3913f9e0fdc04fa2ab31379c9048ad9a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:42.428992 25465 ts_tablet_manager.cc:1909] T 6844411c7a99471f8f5bbe36ed89f91e P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:42.429302 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 273244b2535e4fcbbd22de0d236715c5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:42.430069 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 6844411c7a99471f8f5bbe36ed89f91e (table rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]) successfully deleted
I20240502 07:58:42.430254 25465 tablet_replica.cc:330] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:42.430406 25465 raft_consensus.cc:2229] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:42.430552 25465 raft_consensus.cc:2258] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.431056 25465 ts_tablet_manager.cc:1875] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.433399 25465 ts_tablet_manager.cc:1888] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.433506 25465 log.cc:1196] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/2dd6748f1e8a4308b89e0329c54d63ac
I20240502 07:58:42.433790 25465 ts_tablet_manager.cc:1909] T 2dd6748f1e8a4308b89e0329c54d63ac P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:42.434463 24015 raft_consensus.cc:482] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:42.434547 24015 raft_consensus.cc:504] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:42.434907 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 2dd6748f1e8a4308b89e0329c54d63ac (table rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]) successfully deleted
I20240502 07:58:42.434998 24015 leader_election.cc:288] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:42.436367 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:42.436925 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 28832d9e18ea41b194cf33d8c7971e23 (DEFAULT_TABLE table=rebalancer_extra_table_0 [id=3913f9e0fdc04fa2ab31379c9048ad9a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:42.437819 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 28832d9e18ea41b194cf33d8c7971e23. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:42.440727 25482 tablet_bootstrap.cc:492] T 273244b2535e4fcbbd22de0d236715c5 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:42.441845 25114 tablet_service.cc:1467] Processing CreateTablet for tablet 273244b2535e4fcbbd22de0d236715c5 (DEFAULT_TABLE table=rebalancer_extra_table_0 [id=3913f9e0fdc04fa2ab31379c9048ad9a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:42.442103 25114 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 273244b2535e4fcbbd22de0d236715c5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:42.442739 25468 tablet_replica.cc:330] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:42.442862 25482 tablet_bootstrap.cc:654] T 273244b2535e4fcbbd22de0d236715c5 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:42.443048 25468 raft_consensus.cc:2229] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:42.443305 25468 raft_consensus.cc:2258] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.443653 25482 log.cc:824] T 273244b2535e4fcbbd22de0d236715c5 P 6d02017318a2461796d431962fe6a537: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:42.444078 25468 ts_tablet_manager.cc:1875] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.445400 25482 tablet_bootstrap.cc:492] T 273244b2535e4fcbbd22de0d236715c5 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:42.445600 25482 ts_tablet_manager.cc:1367] T 273244b2535e4fcbbd22de0d236715c5 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.005s user 0.005s sys 0.000s
I20240502 07:58:42.447407 25468 ts_tablet_manager.cc:1888] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.448845 25482 raft_consensus.cc:348] T 273244b2535e4fcbbd22de0d236715c5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:42.449081 25482 raft_consensus.cc:374] T 273244b2535e4fcbbd22de0d236715c5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:42.449172 25482 raft_consensus.cc:729] T 273244b2535e4fcbbd22de0d236715c5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:42.449445 25468 log.cc:1196] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/2dd6748f1e8a4308b89e0329c54d63ac
I20240502 07:58:42.449600 25482 consensus_queue.cc:260] T 273244b2535e4fcbbd22de0d236715c5 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:42.450363 25468 ts_tablet_manager.cc:1909] T 2dd6748f1e8a4308b89e0329c54d63ac P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:42.450436 25482 ts_tablet_manager.cc:1398] T 273244b2535e4fcbbd22de0d236715c5 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.005s user 0.004s sys 0.000s
I20240502 07:58:42.451294 25400 tablet.cc:1722] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:42.452217 25468 tablet_replica.cc:330] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:42.451944 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "273244b2535e4fcbbd22de0d236715c5" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
W20240502 07:58:42.454509 22915 leader_election.cc:341] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): Illegal state: must be running to vote when last-logged opid is not known
I20240502 07:58:42.454682 25468 raft_consensus.cc:2229] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:42.454809 25468 raft_consensus.cc:2258] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.454879 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 2dd6748f1e8a4308b89e0329c54d63ac (table rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]) successfully deleted
I20240502 07:58:42.455461 25468 ts_tablet_manager.cc:1875] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.457306 25468 ts_tablet_manager.cc:1888] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.457852 25326 tablet_bootstrap.cc:492] T e801bc67a63d4b249b12f9cbd9567582 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:42.457943 25326 ts_tablet_manager.cc:1367] T e801bc67a63d4b249b12f9cbd9567582 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.041s user 0.001s sys 0.000s
I20240502 07:58:42.458288 25326 raft_consensus.cc:348] T e801bc67a63d4b249b12f9cbd9567582 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:42.458396 25485 tablet_bootstrap.cc:492] T 273244b2535e4fcbbd22de0d236715c5 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:42.458415 25326 raft_consensus.cc:374] T e801bc67a63d4b249b12f9cbd9567582 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:42.458462 25326 raft_consensus.cc:729] T e801bc67a63d4b249b12f9cbd9567582 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:42.458501 25468 log.cc:1196] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/6844411c7a99471f8f5bbe36ed89f91e
I20240502 07:58:42.458588 25326 consensus_queue.cc:260] T e801bc67a63d4b249b12f9cbd9567582 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:42.458755 25468 ts_tablet_manager.cc:1909] T 6844411c7a99471f8f5bbe36ed89f91e P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:42.458907 25326 ts_tablet_manager.cc:1398] T e801bc67a63d4b249b12f9cbd9567582 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:42.457554 25475 tablet_bootstrap.cc:492] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:42.459909 25475 tablet_bootstrap.cc:654] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:42.460105 25401 tablet.cc:1722] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:42.460091 25468 tablet_replica.cc:330] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:42.460340 25468 raft_consensus.cc:2229] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:42.460451 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 6844411c7a99471f8f5bbe36ed89f91e (table rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]) successfully deleted
I20240502 07:58:42.460479 25468 raft_consensus.cc:2258] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.461040 25485 tablet_bootstrap.cc:654] T 273244b2535e4fcbbd22de0d236715c5 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:42.461472 25468 ts_tablet_manager.cc:1875] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.462026 25402 tablet.cc:1722] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:42.462106 22442 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=a2b784b31a5b43b68c8b51571c83f116] alter complete (version 3)
I20240502 07:58:42.463104 25485 log.cc:824] T 273244b2535e4fcbbd22de0d236715c5 P 4654231dfe994c1a9d2e37d9c3d75699: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:42.463284 25487 tablet_bootstrap.cc:492] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:42.465123 25468 ts_tablet_manager.cc:1888] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.465214 25468 log.cc:1196] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/207d85c84a6e4929b4842990860c65e6
I20240502 07:58:42.465456 25468 ts_tablet_manager.cc:1909] T 207d85c84a6e4929b4842990860c65e6 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:42.466094 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "273244b2535e4fcbbd22de0d236715c5" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:42.466356 24872 raft_consensus.cc:2454] T 273244b2535e4fcbbd22de0d236715c5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:42.467003 22912 leader_election.cc:302] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 6d02017318a2461796d431962fe6a537; no voters: 4654231dfe994c1a9d2e37d9c3d75699
I20240502 07:58:42.467471 25485 tablet_bootstrap.cc:492] T 273244b2535e4fcbbd22de0d236715c5 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:42.467522 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 207d85c84a6e4929b4842990860c65e6 (table rebalance_test_table_0 [id=adca0d0860134bb1bb90377547f0654d]) successfully deleted
I20240502 07:58:42.467638 25485 ts_tablet_manager.cc:1367] T 273244b2535e4fcbbd22de0d236715c5 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.009s user 0.004s sys 0.000s
I20240502 07:58:42.468022 25487 tablet_bootstrap.cc:654] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:42.468969 25487 log.cc:824] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e: Log is configured to *not* fsync() on all Append() calls
I20240502 07:58:42.470031 25485 raft_consensus.cc:348] T 273244b2535e4fcbbd22de0d236715c5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:42.470193 25485 raft_consensus.cc:374] T 273244b2535e4fcbbd22de0d236715c5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:42.470239 25485 raft_consensus.cc:729] T 273244b2535e4fcbbd22de0d236715c5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:42.470603 25485 consensus_queue.cc:260] T 273244b2535e4fcbbd22de0d236715c5 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:42.470896 25487 tablet_bootstrap.cc:492] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:42.471065 25487 ts_tablet_manager.cc:1367] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.000s
I20240502 07:58:42.471177 25181 heartbeater.cc:503] Master 127.16.85.190:39083 was elected leader, sending a full tablet report...
I20240502 07:58:42.471422 25485 ts_tablet_manager.cc:1398] T 273244b2535e4fcbbd22de0d236715c5 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.004s user 0.003s sys 0.000s
I20240502 07:58:42.471617 25485 tablet_bootstrap.cc:492] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:42.472463 25485 tablet_bootstrap.cc:654] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:42.472808 24015 raft_consensus.cc:2781] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:42.472882 24015 raft_consensus.cc:482] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:42.472918 24015 raft_consensus.cc:3037] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:42.473942 24015 raft_consensus.cc:504] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:42.474280 24015 leader_election.cc:288] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:42.474179 25487 raft_consensus.cc:348] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:42.474478 25487 raft_consensus.cc:374] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:42.474531 25487 raft_consensus.cc:729] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:42.474601 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "273244b2535e4fcbbd22de0d236715c5" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:42.474709 24872 raft_consensus.cc:3037] T 273244b2535e4fcbbd22de0d236715c5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:42.474833 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "273244b2535e4fcbbd22de0d236715c5" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:42.474936 25135 raft_consensus.cc:3037] T 273244b2535e4fcbbd22de0d236715c5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:42.474956 25487 consensus_queue.cc:260] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:42.475689 25487 ts_tablet_manager.cc:1398] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.005s user 0.005s sys 0.000s
I20240502 07:58:42.475730 24872 raft_consensus.cc:2454] T 273244b2535e4fcbbd22de0d236715c5 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:42.475884 25135 raft_consensus.cc:2454] T 273244b2535e4fcbbd22de0d236715c5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:42.476032 25050 heartbeater.cc:503] Master 127.16.85.190:39083 was elected leader, sending a full tablet report...
I20240502 07:58:42.476303 25475 tablet_bootstrap.cc:492] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:42.476380 25475 ts_tablet_manager.cc:1367] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.019s user 0.001s sys 0.000s
I20240502 07:58:42.476552 22915 leader_election.cc:302] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 4654231dfe994c1a9d2e37d9c3d75699; no voters:
I20240502 07:58:42.476714 24015 raft_consensus.cc:2781] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:42.476774 24015 raft_consensus.cc:686] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:42.476895 24015 consensus_queue.cc:237] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:42.478658 25475 raft_consensus.cc:348] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:42.478760 25475 raft_consensus.cc:374] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:42.478823 25475 raft_consensus.cc:729] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:42.478963 25475 consensus_queue.cc:260] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:42.479328 25475 ts_tablet_manager.cc:1398] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:42.479590 22442 catalog_manager.cc:5484] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:42.482239 25485 tablet_bootstrap.cc:492] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:42.482316 25485 ts_tablet_manager.cc:1367] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.000s
I20240502 07:58:42.482909 25485 raft_consensus.cc:348] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:42.482987 25485 raft_consensus.cc:374] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:42.483008 25485 raft_consensus.cc:729] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:42.483829 25485 consensus_queue.cc:260] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:42.484126 25485 ts_tablet_manager.cc:1398] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
W20240502 07:58:42.535441 25051 tablet.cc:2369] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 07:58:42.566890 22439 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:42.567054 22439 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:42.569036 22439 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 43ff363342a547afa9edd078f1dfe6f2
I20240502 07:58:42.569772 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet 43ff363342a547afa9edd078f1dfe6f2 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:42.569829 25235 tablet_service.cc:1514] Processing DeleteTablet for tablet 43ff363342a547afa9edd078f1dfe6f2 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:42.569936 25465 tablet_replica.cc:330] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:42.570052 25465 raft_consensus.cc:2229] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:42.570166 25465 raft_consensus.cc:2258] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.570183 22828 tablet_service.cc:1514] Processing DeleteTablet for tablet 43ff363342a547afa9edd078f1dfe6f2 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:42.570330 25467 tablet_replica.cc:330] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:42.570498 25467 raft_consensus.cc:2229] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:42.570613 25467 raft_consensus.cc:2258] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.570816 25465 ts_tablet_manager.cc:1875] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.572788 25465 ts_tablet_manager.cc:1888] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:42.572839 25465 log.cc:1196] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/43ff363342a547afa9edd078f1dfe6f2
I20240502 07:58:42.573058 25465 ts_tablet_manager.cc:1909] T 43ff363342a547afa9edd078f1dfe6f2 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:42.573606 25467 ts_tablet_manager.cc:1875] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.573726 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 43ff363342a547afa9edd078f1dfe6f2 (table rebalancer_dynamic_table_new_name [id=a2b784b31a5b43b68c8b51571c83f116]) successfully deleted
I20240502 07:58:42.575402 25467 ts_tablet_manager.cc:1888] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:42.575471 25467 log.cc:1196] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/43ff363342a547afa9edd078f1dfe6f2
I20240502 07:58:42.575688 25467 ts_tablet_manager.cc:1909] T 43ff363342a547afa9edd078f1dfe6f2 P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:42.576231 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 43ff363342a547afa9edd078f1dfe6f2 (table rebalancer_dynamic_table_new_name [id=a2b784b31a5b43b68c8b51571c83f116]) successfully deleted
I20240502 07:58:42.576918 25527 tablet_replica.cc:330] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:42.577225 25527 raft_consensus.cc:2229] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:42.577569 25527 raft_consensus.cc:2258] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.578698 25527 ts_tablet_manager.cc:1875] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.580451 25527 ts_tablet_manager.cc:1888] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:42.580519 25527 log.cc:1196] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/43ff363342a547afa9edd078f1dfe6f2
I20240502 07:58:42.580817 25527 ts_tablet_manager.cc:1909] T 43ff363342a547afa9edd078f1dfe6f2 P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:58:42.581631 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet 43ff363342a547afa9edd078f1dfe6f2 (table rebalancer_dynamic_table_new_name [id=a2b784b31a5b43b68c8b51571c83f116]) successfully deleted
I20240502 07:58:42.612027 24714 raft_consensus.cc:482] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:42.612149 24714 raft_consensus.cc:504] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:42.612664 24714 leader_election.cc:288] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:42.613029 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e801bc67a63d4b249b12f9cbd9567582" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:42.613173 22715 raft_consensus.cc:2454] T e801bc67a63d4b249b12f9cbd9567582 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:42.613438 22519 leader_election.cc:302] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:42.613631 24714 raft_consensus.cc:2781] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
W20240502 07:58:42.613622 24919 tablet.cc:2369] T 273244b2535e4fcbbd22de0d236715c5 P 6d02017318a2461796d431962fe6a537: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 07:58:42.613687 24714 raft_consensus.cc:482] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:42.613729 24714 raft_consensus.cc:3037] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:42.614639 24714 raft_consensus.cc:504] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:42.614961 24714 leader_election.cc:288] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:42.615298 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e801bc67a63d4b249b12f9cbd9567582" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:42.615383 22715 raft_consensus.cc:3037] T e801bc67a63d4b249b12f9cbd9567582 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:42.616235 22715 raft_consensus.cc:2454] T e801bc67a63d4b249b12f9cbd9567582 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:42.616492 22519 leader_election.cc:302] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:42.616624 24714 raft_consensus.cc:2781] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:42.616715 24714 raft_consensus.cc:686] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:42.616742 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e801bc67a63d4b249b12f9cbd9567582" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:42.616838 24714 consensus_queue.cc:237] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:42.616925 25265 raft_consensus.cc:3037] T e801bc67a63d4b249b12f9cbd9567582 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:42.617756 25265 raft_consensus.cc:2454] T e801bc67a63d4b249b12f9cbd9567582 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:42.616724 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e801bc67a63d4b249b12f9cbd9567582" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:42.617969 25266 raft_consensus.cc:2362] T e801bc67a63d4b249b12f9cbd9567582 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1. Re-sending same reply.
I20240502 07:58:42.618844 22439 catalog_manager.cc:5484] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:42.677256 22439 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:42.680552 25493 raft_consensus.cc:482] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:42.680759 25493 raft_consensus.cc:504] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:42.681931 25493 leader_election.cc:288] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:42.682103 24984 tablet_service.cc:1467] Processing CreateTablet for tablet b74edd3d45a34dd3b27ed5656cb33d47 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=245285f066874a93ad3d78082f078742]), partition=
I20240502 07:58:42.682356 24984 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b74edd3d45a34dd3b27ed5656cb33d47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:42.682982 25235 tablet_service.cc:1467] Processing CreateTablet for tablet b74edd3d45a34dd3b27ed5656cb33d47 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=245285f066874a93ad3d78082f078742]), partition=
I20240502 07:58:42.683243 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b74edd3d45a34dd3b27ed5656cb33d47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:42.683666 24852 tablet_service.cc:1467] Processing CreateTablet for tablet b74edd3d45a34dd3b27ed5656cb33d47 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=245285f066874a93ad3d78082f078742]), partition=
I20240502 07:58:42.683882 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b74edd3d45a34dd3b27ed5656cb33d47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:42.684391 25487 tablet_bootstrap.cc:492] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:42.685307 25487 tablet_bootstrap.cc:654] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:42.685806 25482 tablet_bootstrap.cc:492] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:42.685886 25329 tablet_bootstrap.cc:492] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:42.686573 25329 tablet_bootstrap.cc:654] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:42.686628 25482 tablet_bootstrap.cc:654] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:42.686960 25487 tablet_bootstrap.cc:492] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:42.687026 25487 ts_tablet_manager.cc:1367] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:42.687536 25487 raft_consensus.cc:348] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:42.687647 25487 raft_consensus.cc:374] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:42.687680 25487 raft_consensus.cc:729] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:42.687788 25487 consensus_queue.cc:260] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:42.688032 25487 ts_tablet_manager.cc:1398] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:42.688171 25329 tablet_bootstrap.cc:492] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:42.688244 25329 ts_tablet_manager.cc:1367] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:42.688714 25329 raft_consensus.cc:348] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:42.688794 25329 raft_consensus.cc:374] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:42.688815 25329 raft_consensus.cc:729] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:42.688917 25329 consensus_queue.cc:260] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:42.688942 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28832d9e18ea41b194cf33d8c7971e23" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:42.689023 25266 raft_consensus.cc:2454] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:42.689442 24938 leader_election.cc:302] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 74f9aa91c2c448f980b512fe2e8d672e, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:42.689757 25493 raft_consensus.cc:2781] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:42.689822 25493 raft_consensus.cc:482] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:42.689851 25493 raft_consensus.cc:3037] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:42.690654 25493 raft_consensus.cc:504] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:42.691011 25493 leader_election.cc:288] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:42.691349 25329 ts_tablet_manager.cc:1398] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:42.691525 25482 tablet_bootstrap.cc:492] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:42.691605 25482 ts_tablet_manager.cc:1367] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:58:42.692142 25482 raft_consensus.cc:348] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:42.692232 25482 raft_consensus.cc:374] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:42.692265 25482 raft_consensus.cc:729] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:42.692380 25482 consensus_queue.cc:260] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:42.692631 25482 ts_tablet_manager.cc:1398] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:42.693347 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28832d9e18ea41b194cf33d8c7971e23" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:42.693482 25135 raft_consensus.cc:2454] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:42.693727 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28832d9e18ea41b194cf33d8c7971e23" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:42.693837 25134 raft_consensus.cc:3037] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:42.694622 25134 raft_consensus.cc:2454] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:42.695622 24939 leader_election.cc:302] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:42.695869 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28832d9e18ea41b194cf33d8c7971e23" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:42.695895 25493 raft_consensus.cc:2781] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:42.695966 25266 raft_consensus.cc:3037] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:42.696084 25493 raft_consensus.cc:686] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Becoming Leader. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Running, Role: LEADER
I20240502 07:58:42.696399 25493 consensus_queue.cc:237] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:42.696808 25266 raft_consensus.cc:2454] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:42.699549 22439 catalog_manager.cc:5484] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: term changed from 0 to 1, leader changed from <none> to 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134). New cstate: current_term: 1 leader_uuid: "74f9aa91c2c448f980b512fe2e8d672e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:42.711246 25182 tablet.cc:2369] T 273244b2535e4fcbbd22de0d236715c5 P 4654231dfe994c1a9d2e37d9c3d75699: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 07:58:42.728979 24852 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:42.729140 25115 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:42.739372 22695 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 07:58:42.742254 25493 raft_consensus.cc:482] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:42.742352 25493 raft_consensus.cc:504] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:42.742822 25493 leader_election.cc:288] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:42.743031 25235 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 07:58:42.743114 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b74edd3d45a34dd3b27ed5656cb33d47" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:42.743237 25266 raft_consensus.cc:2454] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:42.743515 24938 leader_election.cc:302] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 74f9aa91c2c448f980b512fe2e8d672e, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:42.743685 25493 raft_consensus.cc:2781] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:42.743745 25493 raft_consensus.cc:482] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:42.743791 25493 raft_consensus.cc:3037] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:42.744091 22958 tablet_service.cc:1429] Tablet server has 19 leaders and 0 scanners
I20240502 07:58:42.744956 25493 raft_consensus.cc:504] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:42.745285 25493 leader_election.cc:288] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:42.745714 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b74edd3d45a34dd3b27ed5656cb33d47" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:42.745819 25266 raft_consensus.cc:3037] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:42.746729 25266 raft_consensus.cc:2454] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:42.747162 24938 leader_election.cc:302] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 74f9aa91c2c448f980b512fe2e8d672e, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:42.747550 25493 raft_consensus.cc:2781] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:42.747566 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b74edd3d45a34dd3b27ed5656cb33d47" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:42.747635 25493 raft_consensus.cc:686] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Becoming Leader. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Running, Role: LEADER
I20240502 07:58:42.747582 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b74edd3d45a34dd3b27ed5656cb33d47" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:42.747704 24872 raft_consensus.cc:2454] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:42.747756 25493 consensus_queue.cc:237] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:42.749903 22828 tablet_service.cc:1429] Tablet server has 14 leaders and 0 scanners
I20240502 07:58:42.750991 22436 catalog_manager.cc:5484] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: term changed from 0 to 1, leader changed from <none> to 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134). New cstate: current_term: 1 leader_uuid: "74f9aa91c2c448f980b512fe2e8d672e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:42.767525 24984 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:42.779238 22566 tablet_service.cc:1429] Tablet server has 16 leaders and 0 scanners
I20240502 07:58:42.829927 22715 consensus_queue.cc:237] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:42.831421 22584 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.831645 22845 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.831827 25557 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.832034 25394 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.834147 25394 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:42.834544 22845 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:42.836385 22584 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:42.837527 22436 catalog_manager.cc:5484] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:42.840551 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e25ba5cc77334a3595a4575fa98476f0. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:42.846412 22584 consensus_queue.cc:237] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:42.847733 22978 raft_consensus.cc:1261] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.848086 22845 raft_consensus.cc:1261] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.848157 24942 consensus_queue.cc:1035] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.848480 25567 consensus_queue.cc:1035] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.850121 24942 raft_consensus.cc:2932] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:42.850593 22978 raft_consensus.cc:2932] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
W20240502 07:58:42.852097 22519 consensus_peers.cc:485] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): Couldn't send request to peer 6d02017318a2461796d431962fe6a537. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ddebb5e20e134841886be53f8f0d5e02. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:42.852785 22845 raft_consensus.cc:2932] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:42.853490 22442 catalog_manager.cc:5484] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:42.861155 22584 consensus_queue.cc:237] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:42.862730 22845 raft_consensus.cc:1261] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.863018 22715 raft_consensus.cc:1261] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.863354 24942 consensus_queue.cc:1035] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.863421 22441 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:42.863708 24942 consensus_queue.cc:1035] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 07:58:42.866524 22522 consensus_peers.cc:485] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): Couldn't send request to peer 4654231dfe994c1a9d2e37d9c3d75699. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: dd815609d7324cc588f02b6cb40cfbfd. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:42.867108 22715 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:42.867516 24714 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:42.867838 22845 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:42.869927 24871 raft_consensus.cc:3037] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:42.871006 24871 raft_consensus.cc:1261] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:42.871001 22441 catalog_manager.cc:5484] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:42.872057 25493 consensus_queue.cc:1035] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:42.876369 22442 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet b74edd3d45a34dd3b27ed5656cb33d47 (table rebalancer_dynamic_table [id=245285f066874a93ad3d78082f078742]). Expected version 1 got 0
I20240502 07:58:42.877050 25266 raft_consensus.cc:1261] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:42.877785 25587 consensus_queue.cc:1035] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:42.878285 25588 tablet.cc:1722] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:42.879384 25588 tablet.cc:1620] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e: MemRowSet was empty: no flush needed.
I20240502 07:58:42.879989 25400 tablet.cc:1722] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:42.880244 25400 tablet.cc:1620] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:42.880863 22442 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet b74edd3d45a34dd3b27ed5656cb33d47 (table rebalancer_dynamic_table [id=245285f066874a93ad3d78082f078742]). Expected version 1 got 0
I20240502 07:58:42.881145 22436 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet b74edd3d45a34dd3b27ed5656cb33d47 (table rebalancer_dynamic_table [id=245285f066874a93ad3d78082f078742]). Expected version 1 got 0
I20240502 07:58:42.881537 25590 tablet.cc:1722] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:42.882066 25590 tablet.cc:1620] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:42.882666 25579 mvcc.cc:205] Tried to move back new op lower bound from 7023152016864378880 to 7023152016375975936. Current Snapshot: MvccSnapshot[applied={T|T < 7023152016864378880}]
I20240502 07:58:42.884001 22845 consensus_queue.cc:237] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:42.884191 22442 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=245285f066874a93ad3d78082f078742] alter complete (version 1)
I20240502 07:58:42.886019 22978 raft_consensus.cc:1261] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.886171 22584 raft_consensus.cc:1261] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.886672 24774 consensus_queue.cc:1035] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.886960 24774 consensus_queue.cc:1035] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.890295 25592 raft_consensus.cc:2932] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:42.893226 22978 raft_consensus.cc:2932] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:42.894124 22436 catalog_manager.cc:5484] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:42.894416 22783 consensus_peers.cc:485] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Couldn't send request to peer 7eced220cf1642f3873fc385b6191e69. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 848fc8384054446aa75a05a52488ea57. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:42.895859 22584 raft_consensus.cc:2932] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:42.907377 22436 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_2" } modify_external_catalogs: true
I20240502 07:58:42.907498 22436 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_2" } modify_external_catalogs: true
I20240502 07:58:42.909859 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet d7a82b58251f498cba0f0134785bfe5b
I20240502 07:58:42.910094 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet be8ac3cda68f4f3a8e07f913094f3b0e
I20240502 07:58:42.910228 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet b9bd4ccf02cf471685c02a402d883bb5
I20240502 07:58:42.910940 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet be8ac3cda68f4f3a8e07f913094f3b0e with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:42.911043 22958 tablet_service.cc:1514] Processing DeleteTablet for tablet d7a82b58251f498cba0f0134785bfe5b with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:42.911057 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet b9bd4ccf02cf471685c02a402d883bb5 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:42.911178 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet d7a82b58251f498cba0f0134785bfe5b with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:42.911232 25466 tablet_replica.cc:330] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:42.911319 25465 tablet_replica.cc:330] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:42.911339 25466 raft_consensus.cc:2229] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:42.911429 25465 raft_consensus.cc:2229] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:42.911461 25466 raft_consensus.cc:2258] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.911612 25465 raft_consensus.cc:2258] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.911947 25466 ts_tablet_manager.cc:1875] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.912039 25465 ts_tablet_manager.cc:1875] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.913972 25465 ts_tablet_manager.cc:1888] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.913978 25466 ts_tablet_manager.cc:1888] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.914031 25465 log.cc:1196] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/d7a82b58251f498cba0f0134785bfe5b
I20240502 07:58:42.914031 25466 log.cc:1196] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/be8ac3cda68f4f3a8e07f913094f3b0e
I20240502 07:58:42.914264 25465 ts_tablet_manager.cc:1909] T d7a82b58251f498cba0f0134785bfe5b P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:42.914274 25466 ts_tablet_manager.cc:1909] T be8ac3cda68f4f3a8e07f913094f3b0e P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:42.911178 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet b9bd4ccf02cf471685c02a402d883bb5 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:42.914812 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet be8ac3cda68f4f3a8e07f913094f3b0e (table rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]) successfully deleted
I20240502 07:58:42.914858 25466 tablet_replica.cc:330] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:42.914978 25466 raft_consensus.cc:2229] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:42.915086 22828 tablet_service.cc:1514] Processing DeleteTablet for tablet be8ac3cda68f4f3a8e07f913094f3b0e with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:42.915128 25466 raft_consensus.cc:2258] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.915243 25467 tablet_replica.cc:330] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:42.915386 25467 raft_consensus.cc:2229] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:42.915549 25467 raft_consensus.cc:2258] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.915580 25466 ts_tablet_manager.cc:1875] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.915750 22564 tablet_service.cc:1514] Processing DeleteTablet for tablet be8ac3cda68f4f3a8e07f913094f3b0e with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:42.915886 25468 tablet_replica.cc:330] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:42.915984 25468 raft_consensus.cc:2229] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:42.915988 25467 ts_tablet_manager.cc:1875] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.916078 25468 raft_consensus.cc:2258] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.916463 25468 ts_tablet_manager.cc:1875] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.917187 25466 ts_tablet_manager.cc:1888] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.917243 25466 log.cc:1196] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/d7a82b58251f498cba0f0134785bfe5b
I20240502 07:58:42.917487 25466 ts_tablet_manager.cc:1909] T d7a82b58251f498cba0f0134785bfe5b P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:42.918115 25466 tablet_replica.cc:330] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:42.918138 25467 ts_tablet_manager.cc:1888] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.918190 25467 log.cc:1196] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/be8ac3cda68f4f3a8e07f913094f3b0e
I20240502 07:58:42.918229 25466 raft_consensus.cc:2229] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:42.918337 25466 raft_consensus.cc:2258] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.918346 25468 ts_tablet_manager.cc:1888] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.918401 25468 log.cc:1196] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/be8ac3cda68f4f3a8e07f913094f3b0e
I20240502 07:58:42.918433 25467 ts_tablet_manager.cc:1909] T be8ac3cda68f4f3a8e07f913094f3b0e P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:42.918632 25468 ts_tablet_manager.cc:1909] T be8ac3cda68f4f3a8e07f913094f3b0e P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:42.918766 25466 ts_tablet_manager.cc:1875] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.918982 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet d7a82b58251f498cba0f0134785bfe5b with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:42.919102 22565 tablet_service.cc:1514] Processing DeleteTablet for tablet b9bd4ccf02cf471685c02a402d883bb5 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:42 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:42.919400 25468 tablet_replica.cc:330] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:42.919533 25468 raft_consensus.cc:2229] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:42.919641 25468 raft_consensus.cc:2258] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.920127 25468 ts_tablet_manager.cc:1875] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.920279 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet be8ac3cda68f4f3a8e07f913094f3b0e (table rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]) successfully deleted
I20240502 07:58:42.920320 25466 ts_tablet_manager.cc:1888] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.920369 25466 log.cc:1196] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/b9bd4ccf02cf471685c02a402d883bb5
I20240502 07:58:42.920472 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet d7a82b58251f498cba0f0134785bfe5b (table rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]) successfully deleted
I20240502 07:58:42.920593 25466 ts_tablet_manager.cc:1909] T b9bd4ccf02cf471685c02a402d883bb5 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:42.920785 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet be8ac3cda68f4f3a8e07f913094f3b0e (table rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]) successfully deleted
I20240502 07:58:42.921160 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet b9bd4ccf02cf471685c02a402d883bb5 (table rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]) successfully deleted
I20240502 07:58:42.921263 25465 tablet_replica.cc:330] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:42.921325 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet d7a82b58251f498cba0f0134785bfe5b (table rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]) successfully deleted
I20240502 07:58:42.921386 25465 raft_consensus.cc:2229] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:42.921514 25465 raft_consensus.cc:2258] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.921737 25468 ts_tablet_manager.cc:1888] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.921779 25468 log.cc:1196] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/d7a82b58251f498cba0f0134785bfe5b
I20240502 07:58:42.921834 25465 ts_tablet_manager.cc:1875] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.921939 25468 ts_tablet_manager.cc:1909] T d7a82b58251f498cba0f0134785bfe5b P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:42.922653 25468 tablet_replica.cc:330] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:42.922772 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet d7a82b58251f498cba0f0134785bfe5b (table rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]) successfully deleted
I20240502 07:58:42.922818 25468 raft_consensus.cc:2229] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:42.922960 25468 raft_consensus.cc:2258] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:42.923326 25468 ts_tablet_manager.cc:1875] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:42.923378 25465 ts_tablet_manager.cc:1888] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.923472 25465 log.cc:1196] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/b9bd4ccf02cf471685c02a402d883bb5
I20240502 07:58:42.923784 25465 ts_tablet_manager.cc:1909] T b9bd4ccf02cf471685c02a402d883bb5 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:42.924381 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet b9bd4ccf02cf471685c02a402d883bb5 (table rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]) successfully deleted
I20240502 07:58:42.924989 25468 ts_tablet_manager.cc:1888] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:42.925065 25468 log.cc:1196] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/b9bd4ccf02cf471685c02a402d883bb5
I20240502 07:58:42.925292 25468 ts_tablet_manager.cc:1909] T b9bd4ccf02cf471685c02a402d883bb5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:42.925801 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet b9bd4ccf02cf471685c02a402d883bb5 (table rebalance_test_table_2 [id=7161fea707654acc8d621576b691a037]) successfully deleted
I20240502 07:58:42.927548 22978 consensus_queue.cc:237] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:42.928906 22845 raft_consensus.cc:1261] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.929232 25495 consensus_queue.cc:1035] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.929530 22715 raft_consensus.cc:1261] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.929859 24081 consensus_queue.cc:1035] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.931504 25495 raft_consensus.cc:2932] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
W20240502 07:58:42.932106 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a8dcc36740be4ca3ac1d43591f7549bd. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:42.933040 22845 raft_consensus.cc:2932] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:42.933982 22715 raft_consensus.cc:2932] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:42.934263 22442 catalog_manager.cc:5484] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:42.941885 22584 consensus_queue.cc:237] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:42.943086 22715 raft_consensus.cc:1261] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.943197 22978 raft_consensus.cc:1261] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:42.943307 22519 consensus_peers.cc:485] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): Couldn't send request to peer 6d02017318a2461796d431962fe6a537. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f1830cb65a294734ad80f274b89ac185. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:42.943536 24942 consensus_queue.cc:1035] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.943858 24942 consensus_queue.cc:1035] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.945506 25578 raft_consensus.cc:2932] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:42.945868 22715 raft_consensus.cc:2932] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:42.945953 22978 raft_consensus.cc:2932] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:42.948572 22436 catalog_manager.cc:5484] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:42.954825 22978 consensus_queue.cc:237] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:42.955049 25495 consensus_queue.cc:1035] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:42.956055 22715 raft_consensus.cc:1261] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.956714 25495 consensus_queue.cc:1035] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.956807 22584 raft_consensus.cc:1261] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.957142 25495 consensus_queue.cc:1035] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 07:58:42.957595 22915 consensus_peers.cc:485] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): Couldn't send request to peer 4654231dfe994c1a9d2e37d9c3d75699. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 73ad3c23d96b43b1894f767ad5db1e2f. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:42.958588 24081 raft_consensus.cc:2932] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:42.959098 22715 raft_consensus.cc:2932] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:42.959282 22584 raft_consensus.cc:2932] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:42.961230 22436 catalog_manager.cc:5484] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:42.965052 25495 consensus_queue.cc:1035] T 273244b2535e4fcbbd22de0d236715c5 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:42.968763 22845 consensus_queue.cc:237] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:42.969995 22584 raft_consensus.cc:1261] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:42.970180 22783 consensus_peers.cc:485] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 -> Peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Couldn't send request to peer 7eced220cf1642f3873fc385b6191e69. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 90480cc47cac4c84994bfcb9bc8a2dbf. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:42.970221 22715 raft_consensus.cc:1261] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.970445 25532 consensus_queue.cc:1035] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.970592 25592 consensus_queue.cc:1035] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.972147 24774 raft_consensus.cc:2932] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:42.972589 22584 raft_consensus.cc:2932] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:42.972589 22715 raft_consensus.cc:2932] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:42.974989 22442 catalog_manager.cc:5484] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:42.982159 22584 consensus_queue.cc:237] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:42.983417 22845 raft_consensus.cc:1261] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.983417 22715 raft_consensus.cc:1261] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:42.983520 22519 consensus_peers.cc:485] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): Couldn't send request to peer 6d02017318a2461796d431962fe6a537. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9e5fe86b44744df9b2e4d430fdec716f. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:42.983721 25567 consensus_queue.cc:1035] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.983862 24942 consensus_queue.cc:1035] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.985466 25578 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:42.985881 22845 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:42.985908 22715 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:42.988368 22441 catalog_manager.cc:5484] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:42.995653 25567 consensus_queue.cc:1035] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:42.995731 22584 consensus_queue.cc:237] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:42.997134 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:42.997376 22978 raft_consensus.cc:1261] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.997776 24942 consensus_queue.cc:1035] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:42.998180 22715 raft_consensus.cc:1261] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:42.998595 25567 consensus_queue.cc:1035] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.000468 24942 consensus_queue.cc:1035] T e801bc67a63d4b249b12f9cbd9567582 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:43.001889 24942 raft_consensus.cc:2932] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:43.004190 22717 raft_consensus.cc:2932] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:43.005404 25588 tablet.cc:1722] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:43.005723 25588 tablet.cc:1620] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e: MemRowSet was empty: no flush needed.
W20240502 07:58:43.006304 22521 consensus_peers.cc:485] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c8c7521bb8414364b927eed84be37621. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:43.006891 25590 tablet.cc:1722] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:43.007182 25590 tablet.cc:1620] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:43.007359 25400 tablet.cc:1722] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:43.007604 25400 tablet.cc:1620] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:43.008260 22442 catalog_manager.cc:5576] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857) does not have the latest schema for tablet b74edd3d45a34dd3b27ed5656cb33d47 (table rebalancer_dynamic_table [id=245285f066874a93ad3d78082f078742]). Expected version 2 got 1
I20240502 07:58:43.012588 22440 catalog_manager.cc:5484] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:43.013005 22441 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=245285f066874a93ad3d78082f078742] alter complete (version 2)
I20240502 07:58:43.016268 22978 raft_consensus.cc:2932] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:43.022965 22584 consensus_queue.cc:237] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:43.024111 22717 raft_consensus.cc:1261] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.024174 22845 raft_consensus.cc:1261] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:43.024330 22522 consensus_peers.cc:485] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): Couldn't send request to peer 4654231dfe994c1a9d2e37d9c3d75699. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 905422dc16e74174b4f88aa87cbee3d3. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:43.024461 24714 consensus_queue.cc:1035] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.024633 25567 consensus_queue.cc:1035] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.026234 24714 raft_consensus.cc:2932] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:43.026523 22717 raft_consensus.cc:2932] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:43.026506 22845 raft_consensus.cc:2932] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:43.029343 22440 catalog_manager.cc:5484] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:43.035648 22978 consensus_queue.cc:237] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:43.036860 22845 raft_consensus.cc:1261] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.036944 22584 raft_consensus.cc:1261] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.037176 24081 consensus_queue.cc:1035] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 07:58:43.037322 22912 consensus_peers.cc:485] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): Couldn't send request to peer 6d02017318a2461796d431962fe6a537. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b22f8e50a7d24dbdab665e47120602ce. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:43.037479 24081 consensus_queue.cc:1035] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.039063 24015 raft_consensus.cc:2932] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:43.039309 22845 raft_consensus.cc:2932] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:43.041142 22584 raft_consensus.cc:2932] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:43.041743 22442 catalog_manager.cc:5484] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:43.049343 22978 consensus_queue.cc:237] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:43.050691 22717 raft_consensus.cc:1261] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.051096 24015 consensus_queue.cc:1035] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.051698 22847 raft_consensus.cc:1261] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.052389 24081 consensus_queue.cc:1035] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.053943 24015 raft_consensus.cc:2932] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
W20240502 07:58:43.054292 22914 consensus_peers.cc:485] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Couldn't send request to peer 7eced220cf1642f3873fc385b6191e69. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 88c1176b415c412fb2c824f277e4d3c8. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:43.054391 22717 raft_consensus.cc:2932] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:43.055405 22845 raft_consensus.cc:2932] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:43.057089 22439 catalog_manager.cc:5484] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:43.064285 22584 consensus_queue.cc:237] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:43.065651 22845 raft_consensus.cc:1261] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.065802 22978 raft_consensus.cc:1261] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:43.065855 22521 consensus_peers.cc:485] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 76c97fb98dd949b9b65fa02d77968cfa. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:43.065958 24714 consensus_queue.cc:1035] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.066102 25578 consensus_queue.cc:1035] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.067621 25567 raft_consensus.cc:2932] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:43.068017 22978 raft_consensus.cc:2932] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:43.068092 22845 raft_consensus.cc:2932] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:43.070490 22442 catalog_manager.cc:5484] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:43.078310 22978 consensus_queue.cc:237] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:43.079514 22584 raft_consensus.cc:1261] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.079612 22717 raft_consensus.cc:1261] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.079871 25670 consensus_queue.cc:1035] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.080035 24015 consensus_queue.cc:1035] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 07:58:43.080184 22915 consensus_peers.cc:485] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): Couldn't send request to peer 4654231dfe994c1a9d2e37d9c3d75699. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1d382df270004c128dc5718be32532ba. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:43.081673 24081 raft_consensus.cc:2932] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:43.082152 22717 raft_consensus.cc:2932] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:43.082314 22584 raft_consensus.cc:2932] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:43.084442 22439 catalog_manager.cc:5484] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:43.091542 22845 consensus_queue.cc:237] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:43.092753 22584 raft_consensus.cc:1261] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.093000 22717 raft_consensus.cc:1261] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.093132 25592 consensus_queue.cc:1035] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.093367 24774 consensus_queue.cc:1035] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.094769 25592 raft_consensus.cc:2932] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:43.095309 22584 raft_consensus.cc:2932] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
W20240502 07:58:43.097402 22781 consensus_peers.cc:485] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): Couldn't send request to peer 6d02017318a2461796d431962fe6a537. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9cf91afea4ed460bb478a031f9520590. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:43.097950 22440 catalog_manager.cc:5484] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:43.098316 22717 raft_consensus.cc:2932] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:43.104436 22978 consensus_queue.cc:237] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:43.105641 22717 raft_consensus.cc:1261] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.105638 22845 raft_consensus.cc:1261] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:43.105865 22914 consensus_peers.cc:485] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Couldn't send request to peer 7eced220cf1642f3873fc385b6191e69. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cfff02438656430686be0f03f5c0b8ab. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:43.105988 25670 consensus_queue.cc:1035] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.106170 24081 consensus_queue.cc:1035] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.107678 25495 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:43.108063 22717 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:43.108143 22845 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:43.110731 22442 catalog_manager.cc:5484] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:43.118809 22717 consensus_queue.cc:237] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:43.119998 22978 raft_consensus.cc:1261] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.120308 22584 raft_consensus.cc:1261] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.120354 25557 consensus_queue.cc:1035] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.120409 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:43.120793 25557 consensus_queue.cc:1035] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.123943 25556 raft_consensus.cc:2932] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:43.124318 22978 raft_consensus.cc:2932] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:43.126626 22439 catalog_manager.cc:5484] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:43.127115 22653 consensus_peers.cc:485] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): Couldn't send request to peer 4654231dfe994c1a9d2e37d9c3d75699. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0e01743578f947cca1db2baec4fdee12. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:43.128329 22584 raft_consensus.cc:2932] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:43.128922 25590 tablet.cc:1722] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:43.129760 25588 tablet.cc:1722] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:43.133608 25400 tablet.cc:1722] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:43.134972 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=245285f066874a93ad3d78082f078742] alter complete (version 3)
I20240502 07:58:43.139621 22978 consensus_queue.cc:237] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:43.140758 22717 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.140807 22845 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:43.141042 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f1fa312fe8e44d23970c5e3ba98c2c58. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:43.141098 25670 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.141233 25669 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.142664 25669 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:43.143102 22717 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:43.144214 22845 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:43.145440 22440 catalog_manager.cc:5484] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:43.237411 22442 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:43.237565 22442 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:43.239769 22442 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet b74edd3d45a34dd3b27ed5656cb33d47
I20240502 07:58:43.240434 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet b74edd3d45a34dd3b27ed5656cb33d47 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:43.240442 24984 tablet_service.cc:1514] Processing DeleteTablet for tablet b74edd3d45a34dd3b27ed5656cb33d47 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:42878
I20240502 07:58:43.240677 25235 tablet_service.cc:1514] Processing DeleteTablet for tablet b74edd3d45a34dd3b27ed5656cb33d47 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:43.240990 25822 tablet_replica.cc:330] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:43.241068 25823 tablet_replica.cc:330] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e: stopping tablet replica
I20240502 07:58:43.241317 25823 raft_consensus.cc:2229] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:43.241317 25822 raft_consensus.cc:2229] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:43.241508 25822 raft_consensus.cc:2258] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.241673 25823 raft_consensus.cc:2258] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.242576 25822 ts_tablet_manager.cc:1875] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.242899 25823 ts_tablet_manager.cc:1875] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.243116 25824 tablet_replica.cc:330] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:43.244228 25824 raft_consensus.cc:2229] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:43.244406 25824 raft_consensus.cc:2258] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.244973 25824 ts_tablet_manager.cc:1875] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.245092 25822 ts_tablet_manager.cc:1888] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:43.245182 25822 log.cc:1196] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/b74edd3d45a34dd3b27ed5656cb33d47
I20240502 07:58:43.245520 25822 ts_tablet_manager.cc:1909] T b74edd3d45a34dd3b27ed5656cb33d47 P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:43.246026 25823 ts_tablet_manager.cc:1888] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:43.246133 25823 log.cc:1196] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/wals/b74edd3d45a34dd3b27ed5656cb33d47
I20240502 07:58:43.246477 25823 ts_tablet_manager.cc:1909] T b74edd3d45a34dd3b27ed5656cb33d47 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting consensus metadata
I20240502 07:58:43.246686 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet b74edd3d45a34dd3b27ed5656cb33d47 (table rebalancer_dynamic_table_new_name [id=245285f066874a93ad3d78082f078742]) successfully deleted
I20240502 07:58:43.246986 25824 ts_tablet_manager.cc:1888] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:43.247128 25824 log.cc:1196] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/b74edd3d45a34dd3b27ed5656cb33d47
I20240502 07:58:43.247380 25824 ts_tablet_manager.cc:1909] T b74edd3d45a34dd3b27ed5656cb33d47 P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:58:43.247570 22430 catalog_manager.cc:4830] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): tablet b74edd3d45a34dd3b27ed5656cb33d47 (table rebalancer_dynamic_table_new_name [id=245285f066874a93ad3d78082f078742]) successfully deleted
I20240502 07:58:43.247908 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet b74edd3d45a34dd3b27ed5656cb33d47 (table rebalancer_dynamic_table_new_name [id=245285f066874a93ad3d78082f078742]) successfully deleted
I20240502 07:58:43.249807 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:43.254577 24852 tablet_service.cc:1467] Processing CreateTablet for tablet a071d3a1489e4d97ae6ceea4b0478459 (DEFAULT_TABLE table=rebalancer_extra_table_2 [id=0c4cae853d5c4dd0a2340f80a1b1b703]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:43.254700 22565 tablet_service.cc:1467] Processing CreateTablet for tablet a071d3a1489e4d97ae6ceea4b0478459 (DEFAULT_TABLE table=rebalancer_extra_table_2 [id=0c4cae853d5c4dd0a2340f80a1b1b703]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:43.254813 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a071d3a1489e4d97ae6ceea4b0478459. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.254956 22565 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a071d3a1489e4d97ae6ceea4b0478459. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.255340 24851 tablet_service.cc:1467] Processing CreateTablet for tablet cfcaa9ee1fbe48ae91796b184ecaf2a3 (DEFAULT_TABLE table=rebalancer_extra_table_2 [id=0c4cae853d5c4dd0a2340f80a1b1b703]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:43.255532 24851 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cfcaa9ee1fbe48ae91796b184ecaf2a3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.256598 25825 tablet_bootstrap.cc:492] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:43.257453 25826 tablet_bootstrap.cc:492] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:43.257571 25825 tablet_bootstrap.cc:654] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:43.257884 24984 tablet_service.cc:1467] Processing CreateTablet for tablet a071d3a1489e4d97ae6ceea4b0478459 (DEFAULT_TABLE table=rebalancer_extra_table_2 [id=0c4cae853d5c4dd0a2340f80a1b1b703]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:43.258037 25115 tablet_service.cc:1467] Processing CreateTablet for tablet cfcaa9ee1fbe48ae91796b184ecaf2a3 (DEFAULT_TABLE table=rebalancer_extra_table_2 [id=0c4cae853d5c4dd0a2340f80a1b1b703]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:43.258131 24984 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a071d3a1489e4d97ae6ceea4b0478459. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.258320 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cfcaa9ee1fbe48ae91796b184ecaf2a3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.258361 25826 tablet_bootstrap.cc:654] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:43.259419 25826 tablet_bootstrap.cc:492] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:43.259490 25826 ts_tablet_manager.cc:1367] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:43.259902 25829 tablet_bootstrap.cc:492] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:43.259925 25826 raft_consensus.cc:348] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:43.260062 25826 raft_consensus.cc:374] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:43.260120 25826 raft_consensus.cc:729] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:43.260274 25826 consensus_queue.cc:260] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:43.260536 25826 ts_tablet_manager.cc:1398] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:43.260576 25114 tablet_service.cc:1467] Processing CreateTablet for tablet 5487e66cb4d8416094a1321cb989f018 (DEFAULT_TABLE table=rebalancer_extra_table_2 [id=0c4cae853d5c4dd0a2340f80a1b1b703]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:43.260761 25826 tablet_bootstrap.cc:492] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:43.260841 25114 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5487e66cb4d8416094a1321cb989f018. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.260915 25235 tablet_service.cc:1467] Processing CreateTablet for tablet cfcaa9ee1fbe48ae91796b184ecaf2a3 (DEFAULT_TABLE table=rebalancer_extra_table_2 [id=0c4cae853d5c4dd0a2340f80a1b1b703]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:43.261160 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cfcaa9ee1fbe48ae91796b184ecaf2a3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.261310 22565 tablet_service.cc:1467] Processing CreateTablet for tablet 5487e66cb4d8416094a1321cb989f018 (DEFAULT_TABLE table=rebalancer_extra_table_2 [id=0c4cae853d5c4dd0a2340f80a1b1b703]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:43.261523 22565 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5487e66cb4d8416094a1321cb989f018. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.261603 25826 tablet_bootstrap.cc:654] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:43.262252 25830 tablet_bootstrap.cc:492] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:43.263599 25830 tablet_bootstrap.cc:654] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:43.263603 25829 tablet_bootstrap.cc:654] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:43.264072 25825 tablet_bootstrap.cc:492] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:43.264153 25825 ts_tablet_manager.cc:1367] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.000s
I20240502 07:58:43.264536 25825 raft_consensus.cc:348] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:43.264597 25826 tablet_bootstrap.cc:492] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:43.264624 25825 raft_consensus.cc:374] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:43.264654 25825 raft_consensus.cc:729] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:43.264667 25826 ts_tablet_manager.cc:1367] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:43.264775 25825 consensus_queue.cc:260] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:43.265041 25825 ts_tablet_manager.cc:1398] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:43.265215 24984 tablet_service.cc:1467] Processing CreateTablet for tablet 5487e66cb4d8416094a1321cb989f018 (DEFAULT_TABLE table=rebalancer_extra_table_2 [id=0c4cae853d5c4dd0a2340f80a1b1b703]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:43.265450 24984 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5487e66cb4d8416094a1321cb989f018. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.265632 25826 raft_consensus.cc:348] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:43.265735 25826 raft_consensus.cc:374] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:43.265770 25826 raft_consensus.cc:729] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:43.265872 25826 consensus_queue.cc:260] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:43.266084 25826 ts_tablet_manager.cc:1398] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:43.266103 25829 tablet_bootstrap.cc:492] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:43.266212 25829 ts_tablet_manager.cc:1367] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:58:43.266593 25829 raft_consensus.cc:348] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:43.266673 25829 raft_consensus.cc:374] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:43.266716 25829 raft_consensus.cc:729] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:43.266855 25829 consensus_queue.cc:260] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:43.267107 25829 ts_tablet_manager.cc:1398] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:43.267269 25829 tablet_bootstrap.cc:492] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:43.268146 25829 tablet_bootstrap.cc:654] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:43.269579 25832 tablet_bootstrap.cc:492] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:43.270478 25832 tablet_bootstrap.cc:654] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:43.270777 25825 tablet_bootstrap.cc:492] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:43.271579 25829 tablet_bootstrap.cc:492] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:43.271579 25832 tablet_bootstrap.cc:492] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:43.271647 25832 ts_tablet_manager.cc:1367] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:43.271662 25829 ts_tablet_manager.cc:1367] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:43.271867 25825 tablet_bootstrap.cc:654] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:43.272051 25829 raft_consensus.cc:348] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.272051 25832 raft_consensus.cc:348] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:43.272133 25829 raft_consensus.cc:374] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:43.272161 25829 raft_consensus.cc:729] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:43.272174 25832 raft_consensus.cc:374] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:43.272209 25832 raft_consensus.cc:729] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:43.272267 25829 consensus_queue.cc:260] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.272364 25832 consensus_queue.cc:260] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:43.272507 25829 ts_tablet_manager.cc:1398] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:43.272270 25587 consensus_queue.cc:1035] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20240502 07:58:43.273257 25825 tablet_bootstrap.cc:492] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:43.273331 25825 ts_tablet_manager.cc:1367] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.001s
I20240502 07:58:43.273743 25825 raft_consensus.cc:348] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.273854 25825 raft_consensus.cc:374] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:43.273890 25825 raft_consensus.cc:729] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:43.273988 25825 consensus_queue.cc:260] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.274258 25825 ts_tablet_manager.cc:1398] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:43.275305 25830 tablet_bootstrap.cc:492] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:43.275398 25830 ts_tablet_manager.cc:1367] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.013s user 0.002s sys 0.000s
I20240502 07:58:43.274242 25832 ts_tablet_manager.cc:1398] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:43.275873 25830 raft_consensus.cc:348] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:43.275974 25830 raft_consensus.cc:374] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:43.276002 25830 raft_consensus.cc:729] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:43.276104 25830 consensus_queue.cc:260] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:43.276352 25830 ts_tablet_manager.cc:1398] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:43.276517 25830 tablet_bootstrap.cc:492] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:43.277487 25830 tablet_bootstrap.cc:654] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:43.278371 25830 tablet_bootstrap.cc:492] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:43.278491 25830 ts_tablet_manager.cc:1367] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:43.278833 25830 raft_consensus.cc:348] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.278935 25830 raft_consensus.cc:374] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:43.278966 25830 raft_consensus.cc:729] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:43.279073 25830 consensus_queue.cc:260] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.279299 25830 ts_tablet_manager.cc:1398] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:43.281334 25840 consensus_queue.cc:1035] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:43.292604 25842 ts_tablet_manager.cc:902] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69: Initiating tablet copy from peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:43.293339 25842 tablet_copy_client.cc:308] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.131:44055
I20240502 07:58:43.293992 22868 tablet_copy_service.cc:140] P 7db4178eed174225bfb24f5f9445ce88: Received BeginTabletCopySession request for tablet 848fc8384054446aa75a05a52488ea57 from peer 7eced220cf1642f3873fc385b6191e69 ({username='slave'} at 127.16.85.136:46167)
I20240502 07:58:43.294095 22868 tablet_copy_service.cc:161] P 7db4178eed174225bfb24f5f9445ce88: Beginning new tablet copy session on tablet 848fc8384054446aa75a05a52488ea57 from peer 7eced220cf1642f3873fc385b6191e69 at {username='slave'} at 127.16.85.136:46167: session id = 7eced220cf1642f3873fc385b6191e69-848fc8384054446aa75a05a52488ea57
I20240502 07:58:43.295056 22868 tablet_copy_source_session.cc:205] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.296355 25842 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 848fc8384054446aa75a05a52488ea57. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.298213 25842 tablet_copy_client.cc:783] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.298360 25842 tablet_copy_client.cc:647] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.299295 25350 raft_consensus.cc:482] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:43.299376 25350 raft_consensus.cc:504] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:43.299813 25350 leader_election.cc:288] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:43.299858 25842 tablet_copy_client.cc:523] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.301306 25842 tablet_bootstrap.cc:492] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:43.304476 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cfcaa9ee1fbe48ae91796b184ecaf2a3" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:43.304466 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cfcaa9ee1fbe48ae91796b184ecaf2a3" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:43.304585 25134 raft_consensus.cc:2454] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:43.304585 24871 raft_consensus.cc:2454] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:43.304860 25202 leader_election.cc:302] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:43.305069 25350 raft_consensus.cc:2781] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:43.305110 25350 raft_consensus.cc:482] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:43.305135 25350 raft_consensus.cc:3037] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:43.305783 25350 raft_consensus.cc:504] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:43.306110 25350 leader_election.cc:288] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:43.306341 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cfcaa9ee1fbe48ae91796b184ecaf2a3" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:43.306442 25134 raft_consensus.cc:3037] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:43.306493 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cfcaa9ee1fbe48ae91796b184ecaf2a3" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:43.306596 24871 raft_consensus.cc:3037] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:43.307209 25134 raft_consensus.cc:2454] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:43.307524 24871 raft_consensus.cc:2454] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:43.307746 25199 leader_election.cc:302] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:43.307920 25350 raft_consensus.cc:2781] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:43.308023 25350 raft_consensus.cc:686] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:43.308141 25350 consensus_queue.cc:237] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:43.309700 22442 catalog_manager.cc:5484] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:43.316468 25851 ts_tablet_manager.cc:902] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:43.317217 25851 tablet_copy_client.cc:308] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:43.320845 22999 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet a8dcc36740be4ca3ac1d43591f7549bd from peer 74f9aa91c2c448f980b512fe2e8d672e ({username='slave'} at 127.16.85.134:46615)
I20240502 07:58:43.320945 22999 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet a8dcc36740be4ca3ac1d43591f7549bd from peer 74f9aa91c2c448f980b512fe2e8d672e at {username='slave'} at 127.16.85.134:46615: session id = 74f9aa91c2c448f980b512fe2e8d672e-a8dcc36740be4ca3ac1d43591f7549bd
I20240502 07:58:43.321907 22999 tablet_copy_source_session.cc:205] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.323247 25851 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a8dcc36740be4ca3ac1d43591f7549bd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.324829 25842 tablet_bootstrap.cc:492] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.325075 25842 tablet_bootstrap.cc:492] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69: Bootstrap complete.
I20240502 07:58:43.325099 25851 tablet_copy_client.cc:783] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.325235 25842 ts_tablet_manager.cc:1367] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.024s user 0.021s sys 0.002s
I20240502 07:58:43.325259 25851 tablet_copy_client.cc:647] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.325791 25842 raft_consensus.cc:348] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:43.325930 25842 raft_consensus.cc:729] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: LEARNER
I20240502 07:58:43.326033 25842 consensus_queue.cc:260] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:43.326298 25842 ts_tablet_manager.cc:1398] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:43.327073 22868 tablet_copy_service.cc:342] P 7db4178eed174225bfb24f5f9445ce88: Request end of tablet copy session 7eced220cf1642f3873fc385b6191e69-848fc8384054446aa75a05a52488ea57 received from {username='slave'} at 127.16.85.136:46167
I20240502 07:58:43.327134 25851 tablet_copy_client.cc:523] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.327169 22868 tablet_copy_service.cc:433] P 7db4178eed174225bfb24f5f9445ce88: ending tablet copy session 7eced220cf1642f3873fc385b6191e69-848fc8384054446aa75a05a52488ea57 on tablet 848fc8384054446aa75a05a52488ea57 with peer 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:43.328282 25851 tablet_bootstrap.cc:492] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:43.341630 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:43.345938 24851 tablet_service.cc:1467] Processing CreateTablet for tablet 4b3baeb1420c4178addd68ce3e1d9606 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=bcb0c0b7c0d345d7ae613152d1a93ccb]), partition=
I20240502 07:58:43.346063 24984 tablet_service.cc:1467] Processing CreateTablet for tablet 4b3baeb1420c4178addd68ce3e1d9606 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=bcb0c0b7c0d345d7ae613152d1a93ccb]), partition=
I20240502 07:58:43.346099 25114 tablet_service.cc:1467] Processing CreateTablet for tablet 4b3baeb1420c4178addd68ce3e1d9606 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=bcb0c0b7c0d345d7ae613152d1a93ccb]), partition=
I20240502 07:58:43.346174 24851 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4b3baeb1420c4178addd68ce3e1d9606. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.346253 24984 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4b3baeb1420c4178addd68ce3e1d9606. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.346297 25114 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4b3baeb1420c4178addd68ce3e1d9606. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.347543 25829 tablet_bootstrap.cc:492] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:43.348086 25830 tablet_bootstrap.cc:492] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:43.348331 25829 tablet_bootstrap.cc:654] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:43.348775 25826 tablet_bootstrap.cc:492] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:43.349318 25829 tablet_bootstrap.cc:492] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:43.349418 25829 ts_tablet_manager.cc:1367] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:43.349802 25829 raft_consensus.cc:348] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.349896 25826 tablet_bootstrap.cc:654] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:43.349905 25829 raft_consensus.cc:374] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:43.349936 25829 raft_consensus.cc:729] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:43.350056 25829 consensus_queue.cc:260] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.350303 25829 ts_tablet_manager.cc:1398] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:43.350879 25830 tablet_bootstrap.cc:654] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:43.351397 25826 tablet_bootstrap.cc:492] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:43.351470 25826 ts_tablet_manager.cc:1367] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240502 07:58:43.351917 25826 raft_consensus.cc:348] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.352051 25826 raft_consensus.cc:374] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:43.352095 25826 raft_consensus.cc:729] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:43.352208 25826 consensus_queue.cc:260] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.352483 25826 ts_tablet_manager.cc:1398] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:43.352759 25842 ts_tablet_manager.cc:902] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69: Initiating tablet copy from peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:43.353207 25842 tablet_copy_client.cc:308] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.131:44055
I20240502 07:58:43.353281 25830 tablet_bootstrap.cc:492] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:43.353348 25830 ts_tablet_manager.cc:1367] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:58:43.353708 25830 raft_consensus.cc:348] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.353780 25830 raft_consensus.cc:374] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:43.353801 25830 raft_consensus.cc:729] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:43.353902 25830 consensus_queue.cc:260] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.354161 25830 ts_tablet_manager.cc:1398] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:43.354265 22868 tablet_copy_service.cc:140] P 7db4178eed174225bfb24f5f9445ce88: Received BeginTabletCopySession request for tablet 90480cc47cac4c84994bfcb9bc8a2dbf from peer 7eced220cf1642f3873fc385b6191e69 ({username='slave'} at 127.16.85.136:46167)
I20240502 07:58:43.354355 22868 tablet_copy_service.cc:161] P 7db4178eed174225bfb24f5f9445ce88: Beginning new tablet copy session on tablet 90480cc47cac4c84994bfcb9bc8a2dbf from peer 7eced220cf1642f3873fc385b6191e69 at {username='slave'} at 127.16.85.136:46167: session id = 7eced220cf1642f3873fc385b6191e69-90480cc47cac4c84994bfcb9bc8a2dbf
I20240502 07:58:43.355180 22868 tablet_copy_source_session.cc:205] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.356257 25842 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 90480cc47cac4c84994bfcb9bc8a2dbf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.358040 25842 tablet_copy_client.cc:783] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.358167 25842 tablet_copy_client.cc:647] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.359221 25842 tablet_copy_client.cc:523] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.360219 25842 tablet_bootstrap.cc:492] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:43.360309 25851 tablet_bootstrap.cc:492] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.360597 25851 tablet_bootstrap.cc:492] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap complete.
I20240502 07:58:43.360802 25851 ts_tablet_manager.cc:1367] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.033s user 0.024s sys 0.000s
I20240502 07:58:43.361241 25851 raft_consensus.cc:348] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:43.361349 25851 raft_consensus.cc:729] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: LEARNER
I20240502 07:58:43.361433 25851 consensus_queue.cc:260] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:43.361722 25851 ts_tablet_manager.cc:1398] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:43.363402 22999 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-a8dcc36740be4ca3ac1d43591f7549bd received from {username='slave'} at 127.16.85.134:46615
I20240502 07:58:43.363489 22999 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-a8dcc36740be4ca3ac1d43591f7549bd on tablet a8dcc36740be4ca3ac1d43591f7549bd with peer 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:43.370332 25840 raft_consensus.cc:482] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:43.370411 25840 raft_consensus.cc:504] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.370733 25840 leader_election.cc:288] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:43.371011 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b3baeb1420c4178addd68ce3e1d9606" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:43.371129 24871 raft_consensus.cc:2454] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:43.371181 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b3baeb1420c4178addd68ce3e1d9606" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:43.371289 25134 raft_consensus.cc:2454] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:43.371529 24936 leader_election.cc:302] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:43.371976 25840 raft_consensus.cc:2781] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:43.372033 25840 raft_consensus.cc:482] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:43.372068 25840 raft_consensus.cc:3037] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:43.372936 25840 raft_consensus.cc:504] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.373286 25840 leader_election.cc:288] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:43.373584 25492 raft_consensus.cc:482] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:43.373674 25492 raft_consensus.cc:504] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.373689 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b3baeb1420c4178addd68ce3e1d9606" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:43.373785 24871 raft_consensus.cc:3037] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:43.374625 24871 raft_consensus.cc:2454] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:43.374670 25492 leader_election.cc:288] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:43.375021 24936 leader_election.cc:302] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:43.375149 25840 raft_consensus.cc:2781] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:43.375207 25840 raft_consensus.cc:686] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Becoming Leader. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Running, Role: LEADER
I20240502 07:58:43.375325 25840 consensus_queue.cc:237] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.375833 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b3baeb1420c4178addd68ce3e1d9606" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:43.375921 25134 raft_consensus.cc:3037] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:43.376772 25134 raft_consensus.cc:2454] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:43.376808 22442 catalog_manager.cc:5484] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: term changed from 0 to 1, leader changed from <none> to 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134). New cstate: current_term: 1 leader_uuid: "74f9aa91c2c448f980b512fe2e8d672e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:43.379923 25851 ts_tablet_manager.cc:902] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e: Initiating tablet copy from peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:43.380347 25851 tablet_copy_client.cc:308] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:37451
I20240502 07:58:43.384042 25842 tablet_bootstrap.cc:492] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.384253 25842 tablet_bootstrap.cc:492] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69: Bootstrap complete.
I20240502 07:58:43.384390 25842 ts_tablet_manager.cc:1367] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.024s user 0.017s sys 0.003s
I20240502 07:58:43.384778 25842 raft_consensus.cc:348] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:43.384902 25842 raft_consensus.cc:729] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: LEARNER
I20240502 07:58:43.384994 25842 consensus_queue.cc:260] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:43.385264 25842 ts_tablet_manager.cc:1398] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.004s sys 0.000s
I20240502 07:58:43.386551 22868 tablet_copy_service.cc:342] P 7db4178eed174225bfb24f5f9445ce88: Request end of tablet copy session 7eced220cf1642f3873fc385b6191e69-90480cc47cac4c84994bfcb9bc8a2dbf received from {username='slave'} at 127.16.85.136:46167
I20240502 07:58:43.386591 22606 tablet_copy_service.cc:140] P 8bea0fe8d8044a2685f92f7a10c9cf85: Received BeginTabletCopySession request for tablet c8c7521bb8414364b927eed84be37621 from peer 74f9aa91c2c448f980b512fe2e8d672e ({username='slave'} at 127.16.85.134:42087)
I20240502 07:58:43.386632 22868 tablet_copy_service.cc:433] P 7db4178eed174225bfb24f5f9445ce88: ending tablet copy session 7eced220cf1642f3873fc385b6191e69-90480cc47cac4c84994bfcb9bc8a2dbf on tablet 90480cc47cac4c84994bfcb9bc8a2dbf with peer 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:43.386693 22606 tablet_copy_service.cc:161] P 8bea0fe8d8044a2685f92f7a10c9cf85: Beginning new tablet copy session on tablet c8c7521bb8414364b927eed84be37621 from peer 74f9aa91c2c448f980b512fe2e8d672e at {username='slave'} at 127.16.85.134:42087: session id = 74f9aa91c2c448f980b512fe2e8d672e-c8c7521bb8414364b927eed84be37621
I20240502 07:58:43.386766 25004 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5487e66cb4d8416094a1321cb989f018" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e" is_pre_election: true
I20240502 07:58:43.386936 25004 raft_consensus.cc:2454] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:43.387316 25070 leader_election.cc:302] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:43.387571 25492 raft_consensus.cc:2781] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:43.387629 25492 raft_consensus.cc:482] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:43.387671 25492 raft_consensus.cc:3037] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:43.387953 22606 tablet_copy_source_session.cc:205] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.388607 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5487e66cb4d8416094a1321cb989f018" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:43.388753 22584 raft_consensus.cc:2454] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:43.388942 25851 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c8c7521bb8414364b927eed84be37621. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.389648 25492 raft_consensus.cc:504] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.389935 25492 leader_election.cc:288] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:43.390179 25004 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5487e66cb4d8416094a1321cb989f018" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e"
I20240502 07:58:43.390262 25004 raft_consensus.cc:3037] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:43.390269 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5487e66cb4d8416094a1321cb989f018" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:43.390341 22584 raft_consensus.cc:3037] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:43.390827 25851 tablet_copy_client.cc:783] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.390955 25851 tablet_copy_client.cc:647] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.391031 22584 raft_consensus.cc:2454] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:43.391117 25004 raft_consensus.cc:2454] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:43.391373 25070 leader_election.cc:302] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:43.391592 25492 raft_consensus.cc:2781] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:43.391768 25492 raft_consensus.cc:686] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:58:43.392022 25492 consensus_queue.cc:237] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:43.392349 25851 tablet_copy_client.cc:523] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.393496 25851 tablet_bootstrap.cc:492] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:43.396067 25877 ts_tablet_manager.cc:902] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:43.396859 25877 tablet_copy_client.cc:308] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:43.397509 22442 catalog_manager.cc:5484] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:43.401517 22999 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet 73ad3c23d96b43b1894f767ad5db1e2f from peer 4654231dfe994c1a9d2e37d9c3d75699 ({username='slave'} at 127.16.85.135:41731)
I20240502 07:58:43.401628 22999 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet 73ad3c23d96b43b1894f767ad5db1e2f from peer 4654231dfe994c1a9d2e37d9c3d75699 at {username='slave'} at 127.16.85.135:41731: session id = 4654231dfe994c1a9d2e37d9c3d75699-73ad3c23d96b43b1894f767ad5db1e2f
I20240502 07:58:43.402287 22999 tablet_copy_source_session.cc:205] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.403381 25877 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 73ad3c23d96b43b1894f767ad5db1e2f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.405174 25877 tablet_copy_client.cc:783] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.405318 25877 tablet_copy_client.cc:647] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.406616 25877 tablet_copy_client.cc:523] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.407765 25877 tablet_bootstrap.cc:492] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:43.413273 25851 tablet_bootstrap.cc:492] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.413532 25851 tablet_bootstrap.cc:492] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap complete.
I20240502 07:58:43.413669 25851 ts_tablet_manager.cc:1367] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.020s user 0.018s sys 0.000s
I20240502 07:58:43.414115 25851 raft_consensus.cc:348] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:43.414219 25851 raft_consensus.cc:729] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: LEARNER
I20240502 07:58:43.414331 25851 consensus_queue.cc:260] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:43.414635 25851 ts_tablet_manager.cc:1398] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 07:58:43.415138 22606 tablet_copy_service.cc:342] P 8bea0fe8d8044a2685f92f7a10c9cf85: Request end of tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-c8c7521bb8414364b927eed84be37621 received from {username='slave'} at 127.16.85.134:42087
I20240502 07:58:43.415215 22606 tablet_copy_service.cc:433] P 8bea0fe8d8044a2685f92f7a10c9cf85: ending tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-c8c7521bb8414364b927eed84be37621 on tablet c8c7521bb8414364b927eed84be37621 with peer 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:43.415315 22436 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_4" } modify_external_catalogs: true
I20240502 07:58:43.415426 22436 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_4" } modify_external_catalogs: true
I20240502 07:58:43.417500 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 4 replicas of tablet 9cf91afea4ed460bb478a031f9520590
I20240502 07:58:43.417893 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet c5853c8107214c618e2ba9d6d983de2f
I20240502 07:58:43.418092 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 4 replicas of tablet 76c97fb98dd949b9b65fa02d77968cfa
I20240502 07:58:43.418397 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet c5853c8107214c618e2ba9d6d983de2f with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:43.418581 25466 tablet_replica.cc:330] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:43.418560 24984 tablet_service.cc:1514] Processing DeleteTablet for tablet 76c97fb98dd949b9b65fa02d77968cfa with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:42878
I20240502 07:58:43.418669 22828 tablet_service.cc:1514] Processing DeleteTablet for tablet 9cf91afea4ed460bb478a031f9520590 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:43.418718 25466 raft_consensus.cc:2229] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:43.418838 25466 raft_consensus.cc:2258] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.418889 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet 76c97fb98dd949b9b65fa02d77968cfa with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:43.419065 25467 tablet_replica.cc:330] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:43.419041 24851 tablet_service.cc:1514] Processing DeleteTablet for tablet 9cf91afea4ed460bb478a031f9520590 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:43.419042 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet 9cf91afea4ed460bb478a031f9520590 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:43.419188 25467 raft_consensus.cc:2229] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:43.419338 25466 ts_tablet_manager.cc:1875] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.419435 25467 raft_consensus.cc:2258] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.419818 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet c5853c8107214c618e2ba9d6d983de2f with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:43.419963 25465 tablet_replica.cc:330] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:43.420083 25465 raft_consensus.cc:2229] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:43.420228 25465 raft_consensus.cc:2258] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.420835 22565 tablet_service.cc:1514] Processing DeleteTablet for tablet 9cf91afea4ed460bb478a031f9520590 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:43.420881 22564 tablet_service.cc:1514] Processing DeleteTablet for tablet 76c97fb98dd949b9b65fa02d77968cfa with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:43.420995 25468 tablet_replica.cc:330] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:43.421124 25468 raft_consensus.cc:2229] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:43.421272 25468 raft_consensus.cc:2258] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
W20240502 07:58:43.421370 22428 catalog_manager.cc:4797] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): delete failed for tablet 9cf91afea4ed460bb478a031f9520590 because the tablet was not found. No further retry: Not found: Tablet not found: 9cf91afea4ed460bb478a031f9520590
I20240502 07:58:43.421418 25466 ts_tablet_manager.cc:1888] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:43.421474 25466 log.cc:1196] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/c5853c8107214c618e2ba9d6d983de2f
I20240502 07:58:43.421541 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet 76c97fb98dd949b9b65fa02d77968cfa with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:43.421876 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet c5853c8107214c618e2ba9d6d983de2f with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:43.421998 25468 ts_tablet_manager.cc:1875] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.422158 25465 ts_tablet_manager.cc:1875] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
W20240502 07:58:43.422248 22430 catalog_manager.cc:4797] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): delete failed for tablet 76c97fb98dd949b9b65fa02d77968cfa because the tablet was not found. No further retry: Not found: Tablet not found: 76c97fb98dd949b9b65fa02d77968cfa
I20240502 07:58:43.421715 25466 ts_tablet_manager.cc:1909] T c5853c8107214c618e2ba9d6d983de2f P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:43.422993 25466 tablet_replica.cc:330] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:43.423130 25466 raft_consensus.cc:2229] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:43.423141 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet c5853c8107214c618e2ba9d6d983de2f (table rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]) successfully deleted
I20240502 07:58:43.423265 25466 raft_consensus.cc:2258] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.423888 25466 ts_tablet_manager.cc:1875] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.424095 25468 ts_tablet_manager.cc:1888] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:43.424153 25468 log.cc:1196] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/9cf91afea4ed460bb478a031f9520590
I20240502 07:58:43.424135 25465 ts_tablet_manager.cc:1888] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:43.424202 25465 log.cc:1196] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/9cf91afea4ed460bb478a031f9520590
I20240502 07:58:43.424412 25468 ts_tablet_manager.cc:1909] T 9cf91afea4ed460bb478a031f9520590 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:43.424451 25465 ts_tablet_manager.cc:1909] T 9cf91afea4ed460bb478a031f9520590 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:43.424938 25468 tablet_replica.cc:330] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:43.425029 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 9cf91afea4ed460bb478a031f9520590 (table rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]) successfully deleted
I20240502 07:58:43.425304 25467 ts_tablet_manager.cc:1875] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.425298 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 9cf91afea4ed460bb478a031f9520590 (table rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]) successfully deleted
I20240502 07:58:43.425470 25465 tablet_replica.cc:330] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:43.425611 25465 raft_consensus.cc:2229] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:43.425750 25465 raft_consensus.cc:2258] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.425755 25466 ts_tablet_manager.cc:1888] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:43.425818 25466 log.cc:1196] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/76c97fb98dd949b9b65fa02d77968cfa
I20240502 07:58:43.426065 25466 ts_tablet_manager.cc:1909] T 76c97fb98dd949b9b65fa02d77968cfa P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:43.426149 25465 ts_tablet_manager.cc:1875] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.426276 25468 raft_consensus.cc:2229] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:43.426499 25468 raft_consensus.cc:2258] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.426993 25468 ts_tablet_manager.cc:1875] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.427191 25467 ts_tablet_manager.cc:1888] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:43.427258 25467 log.cc:1196] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/9cf91afea4ed460bb478a031f9520590
I20240502 07:58:43.427510 25467 ts_tablet_manager.cc:1909] T 9cf91afea4ed460bb478a031f9520590 P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:43.427865 25465 ts_tablet_manager.cc:1888] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:43.427917 25465 log.cc:1196] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/c5853c8107214c618e2ba9d6d983de2f
I20240502 07:58:43.428104 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 9cf91afea4ed460bb478a031f9520590 (table rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]) successfully deleted
I20240502 07:58:43.428139 25465 ts_tablet_manager.cc:1909] T c5853c8107214c618e2ba9d6d983de2f P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:43.428397 25468 ts_tablet_manager.cc:1888] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:43.428459 25468 log.cc:1196] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/76c97fb98dd949b9b65fa02d77968cfa
I20240502 07:58:43.428691 25468 ts_tablet_manager.cc:1909] T 76c97fb98dd949b9b65fa02d77968cfa P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:43.428875 25851 ts_tablet_manager.cc:902] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:43.428851 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet c5853c8107214c618e2ba9d6d983de2f (table rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]) successfully deleted
I20240502 07:58:43.428933 25467 tablet_replica.cc:330] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:43.428851 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 76c97fb98dd949b9b65fa02d77968cfa (table rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]) successfully deleted
I20240502 07:58:43.429052 25467 raft_consensus.cc:2229] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:43.429194 25467 raft_consensus.cc:2258] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.429258 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 76c97fb98dd949b9b65fa02d77968cfa (table rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]) successfully deleted
I20240502 07:58:43.429315 25851 tablet_copy_client.cc:308] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33387
I20240502 07:58:43.429582 25468 tablet_replica.cc:330] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:43.429708 25468 raft_consensus.cc:2229] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:43.429812 25467 ts_tablet_manager.cc:1875] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.429965 25468 raft_consensus.cc:2258] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.430377 25468 ts_tablet_manager.cc:1875] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.431738 25467 ts_tablet_manager.cc:1888] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:43.431805 25467 log.cc:1196] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/76c97fb98dd949b9b65fa02d77968cfa
I20240502 07:58:43.431802 25468 ts_tablet_manager.cc:1888] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:43.431905 25468 log.cc:1196] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/c5853c8107214c618e2ba9d6d983de2f
I20240502 07:58:43.432044 25467 ts_tablet_manager.cc:1909] T 76c97fb98dd949b9b65fa02d77968cfa P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:43.432168 25468 ts_tablet_manager.cc:1909] T c5853c8107214c618e2ba9d6d983de2f P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:43.432598 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 76c97fb98dd949b9b65fa02d77968cfa (table rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]) successfully deleted
I20240502 07:58:43.432746 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet c5853c8107214c618e2ba9d6d983de2f (table rebalance_test_table_4 [id=7ace8c3745324ce49d1ce373ca5116bb]) successfully deleted
I20240502 07:58:43.433969 22737 tablet_copy_service.cc:140] P 725969d490a04fdfb3ba890c3f5096e3: Received BeginTabletCopySession request for tablet e25ba5cc77334a3595a4575fa98476f0 from peer 74f9aa91c2c448f980b512fe2e8d672e ({username='slave'} at 127.16.85.134:53317)
I20240502 07:58:43.434101 22737 tablet_copy_service.cc:161] P 725969d490a04fdfb3ba890c3f5096e3: Beginning new tablet copy session on tablet e25ba5cc77334a3595a4575fa98476f0 from peer 74f9aa91c2c448f980b512fe2e8d672e at {username='slave'} at 127.16.85.134:53317: session id = 74f9aa91c2c448f980b512fe2e8d672e-e25ba5cc77334a3595a4575fa98476f0
I20240502 07:58:43.435096 22737 tablet_copy_source_session.cc:205] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.436378 25851 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e25ba5cc77334a3595a4575fa98476f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.438581 25851 tablet_copy_client.cc:783] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.438761 25851 tablet_copy_client.cc:647] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.440014 25851 tablet_copy_client.cc:523] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.441246 25851 tablet_bootstrap.cc:492] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:43.444829 25883 ts_tablet_manager.cc:902] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699: Initiating tablet copy from peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:43.445353 25883 tablet_copy_client.cc:308] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:37451
I20240502 07:58:43.446363 22606 tablet_copy_service.cc:140] P 8bea0fe8d8044a2685f92f7a10c9cf85: Received BeginTabletCopySession request for tablet dd815609d7324cc588f02b6cb40cfbfd from peer 4654231dfe994c1a9d2e37d9c3d75699 ({username='slave'} at 127.16.85.135:41701)
I20240502 07:58:43.446455 22606 tablet_copy_service.cc:161] P 8bea0fe8d8044a2685f92f7a10c9cf85: Beginning new tablet copy session on tablet dd815609d7324cc588f02b6cb40cfbfd from peer 4654231dfe994c1a9d2e37d9c3d75699 at {username='slave'} at 127.16.85.135:41701: session id = 4654231dfe994c1a9d2e37d9c3d75699-dd815609d7324cc588f02b6cb40cfbfd
I20240502 07:58:43.447216 22606 tablet_copy_source_session.cc:205] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.447978 25883 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dd815609d7324cc588f02b6cb40cfbfd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.449949 25883 tablet_copy_client.cc:783] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.450116 25883 tablet_copy_client.cc:647] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.450452 25898 ts_tablet_manager.cc:902] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537: Initiating tablet copy from peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:43.451114 25877 tablet_bootstrap.cc:492] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.451275 25898 tablet_copy_client.cc:308] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:37451
I20240502 07:58:43.451558 25877 tablet_bootstrap.cc:492] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap complete.
I20240502 07:58:43.451835 25877 ts_tablet_manager.cc:1367] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.044s user 0.022s sys 0.006s
I20240502 07:58:43.452210 25888 ts_tablet_manager.cc:902] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537: Initiating tablet copy from peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:43.452322 25877 raft_consensus.cc:348] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:43.452484 25877 raft_consensus.cc:729] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: LEARNER
I20240502 07:58:43.452673 25877 consensus_queue.cc:260] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:43.453670 25883 tablet_copy_client.cc:523] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.453776 25877 ts_tablet_manager.cc:1398] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:43.454241 22999 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-73ad3c23d96b43b1894f767ad5db1e2f received from {username='slave'} at 127.16.85.135:41731
I20240502 07:58:43.454308 22999 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-73ad3c23d96b43b1894f767ad5db1e2f on tablet 73ad3c23d96b43b1894f767ad5db1e2f with peer 4654231dfe994c1a9d2e37d9c3d75699
I20240502 07:58:43.455576 25883 tablet_bootstrap.cc:492] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:43.458175 25888 tablet_copy_client.cc:308] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:37451
I20240502 07:58:43.464141 22606 tablet_copy_service.cc:140] P 8bea0fe8d8044a2685f92f7a10c9cf85: Received BeginTabletCopySession request for tablet ddebb5e20e134841886be53f8f0d5e02 from peer 6d02017318a2461796d431962fe6a537 ({username='slave'} at 127.16.85.133:57895)
I20240502 07:58:43.464184 22605 tablet_copy_service.cc:140] P 8bea0fe8d8044a2685f92f7a10c9cf85: Received BeginTabletCopySession request for tablet f1830cb65a294734ad80f274b89ac185 from peer 6d02017318a2461796d431962fe6a537 ({username='slave'} at 127.16.85.133:57895)
I20240502 07:58:43.464227 22606 tablet_copy_service.cc:161] P 8bea0fe8d8044a2685f92f7a10c9cf85: Beginning new tablet copy session on tablet ddebb5e20e134841886be53f8f0d5e02 from peer 6d02017318a2461796d431962fe6a537 at {username='slave'} at 127.16.85.133:57895: session id = 6d02017318a2461796d431962fe6a537-ddebb5e20e134841886be53f8f0d5e02
I20240502 07:58:43.465102 22606 tablet_copy_source_session.cc:205] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.465461 22605 tablet_copy_service.cc:161] P 8bea0fe8d8044a2685f92f7a10c9cf85: Beginning new tablet copy session on tablet f1830cb65a294734ad80f274b89ac185 from peer 6d02017318a2461796d431962fe6a537 at {username='slave'} at 127.16.85.133:57895: session id = 6d02017318a2461796d431962fe6a537-f1830cb65a294734ad80f274b89ac185
I20240502 07:58:43.466287 22605 tablet_copy_source_session.cc:205] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.467161 25851 tablet_bootstrap.cc:492] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.467442 25851 tablet_bootstrap.cc:492] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap complete.
I20240502 07:58:43.467660 25851 ts_tablet_manager.cc:1367] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.026s user 0.025s sys 0.000s
I20240502 07:58:43.468144 25851 raft_consensus.cc:348] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:43.468288 25851 raft_consensus.cc:729] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: LEARNER
I20240502 07:58:43.468407 25851 consensus_queue.cc:260] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:43.468724 25851 ts_tablet_manager.cc:1398] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:43.469388 22737 tablet_copy_service.cc:342] P 725969d490a04fdfb3ba890c3f5096e3: Request end of tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-e25ba5cc77334a3595a4575fa98476f0 received from {username='slave'} at 127.16.85.134:53317
I20240502 07:58:43.469476 22737 tablet_copy_service.cc:433] P 725969d490a04fdfb3ba890c3f5096e3: ending tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-e25ba5cc77334a3595a4575fa98476f0 on tablet e25ba5cc77334a3595a4575fa98476f0 with peer 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:43.472776 25888 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f1830cb65a294734ad80f274b89ac185. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.473353 25898 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ddebb5e20e134841886be53f8f0d5e02. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.475214 25888 tablet_copy_client.cc:783] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.475418 25888 tablet_copy_client.cc:647] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.475441 25898 tablet_copy_client.cc:783] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.475680 25898 tablet_copy_client.cc:647] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.477324 25898 tablet_copy_client.cc:523] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.477324 25888 tablet_copy_client.cc:523] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.478439 25898 tablet_bootstrap.cc:492] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:43.478648 25888 tablet_bootstrap.cc:492] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:43.479718 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:43.481720 25883 tablet_bootstrap.cc:492] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.482081 25883 tablet_bootstrap.cc:492] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap complete.
I20240502 07:58:43.482407 25629 raft_consensus.cc:482] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:43.482508 25883 ts_tablet_manager.cc:1367] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.027s user 0.026s sys 0.000s
I20240502 07:58:43.482486 25629 raft_consensus.cc:504] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:43.482929 25883 raft_consensus.cc:348] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:43.483037 25883 raft_consensus.cc:729] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: LEARNER
I20240502 07:58:43.483143 25883 consensus_queue.cc:260] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:43.483485 25883 ts_tablet_manager.cc:1398] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:43.483525 25629 leader_election.cc:288] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:43.483839 25134 raft_consensus.cc:1261] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:43.484189 22606 tablet_copy_service.cc:342] P 8bea0fe8d8044a2685f92f7a10c9cf85: Request end of tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-dd815609d7324cc588f02b6cb40cfbfd received from {username='slave'} at 127.16.85.135:41701
I20240502 07:58:43.484258 22606 tablet_copy_service.cc:433] P 8bea0fe8d8044a2685f92f7a10c9cf85: ending tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-dd815609d7324cc588f02b6cb40cfbfd on tablet dd815609d7324cc588f02b6cb40cfbfd with peer 4654231dfe994c1a9d2e37d9c3d75699
I20240502 07:58:43.484586 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a071d3a1489e4d97ae6ceea4b0478459" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:43.484742 22584 raft_consensus.cc:2454] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:43.484791 24871 raft_consensus.cc:1261] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:43.485299 25840 consensus_queue.cc:1035] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:43.485630 24808 leader_election.cc:302] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:43.485854 25629 raft_consensus.cc:2781] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:43.485917 25629 raft_consensus.cc:482] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:43.485962 25629 raft_consensus.cc:3037] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:43.485976 25840 consensus_queue.cc:1035] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:43.486605 25629 raft_consensus.cc:504] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:43.486843 25629 leader_election.cc:288] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:43.488487 25004 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a071d3a1489e4d97ae6ceea4b0478459" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e" is_pre_election: true
I20240502 07:58:43.488535 25003 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a071d3a1489e4d97ae6ceea4b0478459" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e"
I20240502 07:58:43.488605 25004 raft_consensus.cc:2454] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:43.489547 22436 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 4b3baeb1420c4178addd68ce3e1d9606 (table rebalancer_dynamic_table [id=bcb0c0b7c0d345d7ae613152d1a93ccb]). Expected version 1 got 0
I20240502 07:58:43.490206 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a071d3a1489e4d97ae6ceea4b0478459" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:43.490312 22584 raft_consensus.cc:3037] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:43.491276 22584 raft_consensus.cc:2454] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:43.491546 24808 leader_election.cc:302] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters: 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:43.491859 25588 tablet.cc:1722] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:43.492010 25629 raft_consensus.cc:2781] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:43.492169 25588 tablet.cc:1620] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e: MemRowSet was empty: no flush needed.
I20240502 07:58:43.492158 22442 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet 4b3baeb1420c4178addd68ce3e1d9606 (table rebalancer_dynamic_table [id=bcb0c0b7c0d345d7ae613152d1a93ccb]). Expected version 1 got 0
I20240502 07:58:43.492174 25629 raft_consensus.cc:686] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:58:43.492383 25629 consensus_queue.cc:237] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:43.494419 22440 catalog_manager.cc:5484] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:43.495397 25627 mvcc.cc:205] Tried to move back new op lower bound from 7023152019384664064 to 7023152018946441216. Current Snapshot: MvccSnapshot[applied={T|T < 7023152019384664064}]
I20240502 07:58:43.495718 22442 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=bcb0c0b7c0d345d7ae613152d1a93ccb] alter complete (version 1)
I20240502 07:58:43.495975 25585 mvcc.cc:205] Tried to move back new op lower bound from 7023152019384664064 to 7023152018946441216. Current Snapshot: MvccSnapshot[applied={T|T < 7023152019384664064}]
I20240502 07:58:43.496089 25590 tablet.cc:1722] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:43.496311 25590 tablet.cc:1620] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:43.496680 22442 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 4b3baeb1420c4178addd68ce3e1d9606 (table rebalancer_dynamic_table [id=bcb0c0b7c0d345d7ae613152d1a93ccb]). Expected version 1 got 0
I20240502 07:58:43.501346 25919 tablet.cc:1722] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:43.502017 25919 tablet.cc:1620] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:43.519932 25888 tablet_bootstrap.cc:492] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.520228 25888 tablet_bootstrap.cc:492] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537: Bootstrap complete.
I20240502 07:58:43.520521 25888 ts_tablet_manager.cc:1367] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.042s user 0.018s sys 0.004s
I20240502 07:58:43.522116 25888 raft_consensus.cc:348] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:43.522226 25888 raft_consensus.cc:729] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: LEARNER
I20240502 07:58:43.522351 25888 consensus_queue.cc:260] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:43.522684 25888 ts_tablet_manager.cc:1398] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:43.523154 22606 tablet_copy_service.cc:342] P 8bea0fe8d8044a2685f92f7a10c9cf85: Request end of tablet copy session 6d02017318a2461796d431962fe6a537-f1830cb65a294734ad80f274b89ac185 received from {username='slave'} at 127.16.85.133:57895
I20240502 07:58:43.523228 22606 tablet_copy_service.cc:433] P 8bea0fe8d8044a2685f92f7a10c9cf85: ending tablet copy session 6d02017318a2461796d431962fe6a537-f1830cb65a294734ad80f274b89ac185 on tablet f1830cb65a294734ad80f274b89ac185 with peer 6d02017318a2461796d431962fe6a537
I20240502 07:58:43.529021 25898 tablet_bootstrap.cc:492] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.529284 25898 tablet_bootstrap.cc:492] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537: Bootstrap complete.
I20240502 07:58:43.529436 25898 ts_tablet_manager.cc:1367] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.051s user 0.027s sys 0.000s
I20240502 07:58:43.529775 25898 raft_consensus.cc:348] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:43.529961 25898 raft_consensus.cc:729] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: LEARNER
I20240502 07:58:43.530076 25898 consensus_queue.cc:260] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:43.530409 25898 ts_tablet_manager.cc:1398] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:43.531021 22606 tablet_copy_service.cc:342] P 8bea0fe8d8044a2685f92f7a10c9cf85: Request end of tablet copy session 6d02017318a2461796d431962fe6a537-ddebb5e20e134841886be53f8f0d5e02 received from {username='slave'} at 127.16.85.133:57895
I20240502 07:58:43.531091 22606 tablet_copy_service.cc:433] P 8bea0fe8d8044a2685f92f7a10c9cf85: ending tablet copy session 6d02017318a2461796d431962fe6a537-ddebb5e20e134841886be53f8f0d5e02 on tablet ddebb5e20e134841886be53f8f0d5e02 with peer 6d02017318a2461796d431962fe6a537
I20240502 07:58:43.538759 25883 ts_tablet_manager.cc:902] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699: Initiating tablet copy from peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:43.539129 25883 tablet_copy_client.cc:308] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:37451
I20240502 07:58:43.539498 22606 tablet_copy_service.cc:140] P 8bea0fe8d8044a2685f92f7a10c9cf85: Received BeginTabletCopySession request for tablet 905422dc16e74174b4f88aa87cbee3d3 from peer 4654231dfe994c1a9d2e37d9c3d75699 ({username='slave'} at 127.16.85.135:41701)
I20240502 07:58:43.539582 22606 tablet_copy_service.cc:161] P 8bea0fe8d8044a2685f92f7a10c9cf85: Beginning new tablet copy session on tablet 905422dc16e74174b4f88aa87cbee3d3 from peer 4654231dfe994c1a9d2e37d9c3d75699 at {username='slave'} at 127.16.85.135:41701: session id = 4654231dfe994c1a9d2e37d9c3d75699-905422dc16e74174b4f88aa87cbee3d3
I20240502 07:58:43.540361 22606 tablet_copy_source_session.cc:205] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.541237 25883 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 905422dc16e74174b4f88aa87cbee3d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.543089 25883 tablet_copy_client.cc:783] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.543221 25883 tablet_copy_client.cc:647] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.544406 25883 tablet_copy_client.cc:523] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.545666 25883 tablet_bootstrap.cc:492] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:43.569201 25883 tablet_bootstrap.cc:492] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.569473 25883 tablet_bootstrap.cc:492] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap complete.
I20240502 07:58:43.569669 25883 ts_tablet_manager.cc:1367] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.024s user 0.023s sys 0.000s
I20240502 07:58:43.570021 25883 raft_consensus.cc:348] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:43.570221 25883 raft_consensus.cc:729] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: LEARNER
I20240502 07:58:43.570336 25883 consensus_queue.cc:260] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:43.570600 25883 ts_tablet_manager.cc:1398] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:43.571043 22606 tablet_copy_service.cc:342] P 8bea0fe8d8044a2685f92f7a10c9cf85: Request end of tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-905422dc16e74174b4f88aa87cbee3d3 received from {username='slave'} at 127.16.85.135:41701
I20240502 07:58:43.571116 22606 tablet_copy_service.cc:433] P 8bea0fe8d8044a2685f92f7a10c9cf85: ending tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-905422dc16e74174b4f88aa87cbee3d3 on tablet 905422dc16e74174b4f88aa87cbee3d3 with peer 4654231dfe994c1a9d2e37d9c3d75699
I20240502 07:58:43.594749 25898 ts_tablet_manager.cc:902] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537: Initiating tablet copy from peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:43.595132 25898 tablet_copy_client.cc:308] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:37451
I20240502 07:58:43.596171 22606 tablet_copy_service.cc:140] P 8bea0fe8d8044a2685f92f7a10c9cf85: Received BeginTabletCopySession request for tablet 9e5fe86b44744df9b2e4d430fdec716f from peer 6d02017318a2461796d431962fe6a537 ({username='slave'} at 127.16.85.133:57895)
I20240502 07:58:43.596266 22606 tablet_copy_service.cc:161] P 8bea0fe8d8044a2685f92f7a10c9cf85: Beginning new tablet copy session on tablet 9e5fe86b44744df9b2e4d430fdec716f from peer 6d02017318a2461796d431962fe6a537 at {username='slave'} at 127.16.85.133:57895: session id = 6d02017318a2461796d431962fe6a537-9e5fe86b44744df9b2e4d430fdec716f
I20240502 07:58:43.597216 22606 tablet_copy_source_session.cc:205] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.598168 25898 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9e5fe86b44744df9b2e4d430fdec716f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.598546 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:43.600090 25898 tablet_copy_client.cc:783] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.600219 25898 tablet_copy_client.cc:647] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.604327 25898 tablet_copy_client.cc:523] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.604737 25588 tablet.cc:1722] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:43.605065 25588 tablet.cc:1620] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e: MemRowSet was empty: no flush needed.
I20240502 07:58:43.605180 25590 tablet.cc:1722] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:43.605471 25590 tablet.cc:1620] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:43.605530 25898 tablet_bootstrap.cc:492] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:43.606627 25919 tablet.cc:1722] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:43.606874 25919 tablet.cc:1620] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:43.611881 22442 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=bcb0c0b7c0d345d7ae613152d1a93ccb] alter complete (version 2)
I20240502 07:58:43.620533 25851 ts_tablet_manager.cc:902] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:43.621006 25851 tablet_copy_client.cc:308] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:43.621413 22999 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet f1fa312fe8e44d23970c5e3ba98c2c58 from peer 74f9aa91c2c448f980b512fe2e8d672e ({username='slave'} at 127.16.85.134:46615)
I20240502 07:58:43.621513 22999 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet f1fa312fe8e44d23970c5e3ba98c2c58 from peer 74f9aa91c2c448f980b512fe2e8d672e at {username='slave'} at 127.16.85.134:46615: session id = 74f9aa91c2c448f980b512fe2e8d672e-f1fa312fe8e44d23970c5e3ba98c2c58
I20240502 07:58:43.621969 25883 ts_tablet_manager.cc:902] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:43.622331 25883 tablet_copy_client.cc:308] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:43.622404 22999 tablet_copy_source_session.cc:205] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.622893 22999 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet 1d382df270004c128dc5718be32532ba from peer 4654231dfe994c1a9d2e37d9c3d75699 ({username='slave'} at 127.16.85.135:41731)
I20240502 07:58:43.622978 22999 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet 1d382df270004c128dc5718be32532ba from peer 4654231dfe994c1a9d2e37d9c3d75699 at {username='slave'} at 127.16.85.135:41731: session id = 4654231dfe994c1a9d2e37d9c3d75699-1d382df270004c128dc5718be32532ba
I20240502 07:58:43.623147 25851 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f1fa312fe8e44d23970c5e3ba98c2c58. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.623663 22999 tablet_copy_source_session.cc:205] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.624933 25851 tablet_copy_client.cc:783] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.625084 25851 tablet_copy_client.cc:647] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.625896 25883 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1d382df270004c128dc5718be32532ba. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.626312 25851 tablet_copy_client.cc:523] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.627297 25851 tablet_bootstrap.cc:492] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:43.627827 25883 tablet_copy_client.cc:783] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.627950 25883 tablet_copy_client.cc:647] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.630100 25883 tablet_copy_client.cc:523] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.631328 25883 tablet_bootstrap.cc:492] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:43.636090 25888 ts_tablet_manager.cc:902] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:43.636556 25888 tablet_copy_client.cc:308] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:43.640614 25842 ts_tablet_manager.cc:902] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:43.641028 25842 tablet_copy_client.cc:308] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:43.643525 25898 tablet_bootstrap.cc:492] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.643857 25898 tablet_bootstrap.cc:492] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537: Bootstrap complete.
I20240502 07:58:43.644101 25898 ts_tablet_manager.cc:1367] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.039s user 0.026s sys 0.000s
I20240502 07:58:43.644556 25898 raft_consensus.cc:348] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:43.644767 25898 raft_consensus.cc:729] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: LEARNER
I20240502 07:58:43.644932 25898 consensus_queue.cc:260] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:43.645718 22999 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet 88c1176b415c412fb2c824f277e4d3c8 from peer 7eced220cf1642f3873fc385b6191e69 ({username='slave'} at 127.16.85.136:40185)
I20240502 07:58:43.645803 22999 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet 88c1176b415c412fb2c824f277e4d3c8 from peer 7eced220cf1642f3873fc385b6191e69 at {username='slave'} at 127.16.85.136:40185: session id = 7eced220cf1642f3873fc385b6191e69-88c1176b415c412fb2c824f277e4d3c8
I20240502 07:58:43.646095 22998 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet b22f8e50a7d24dbdab665e47120602ce from peer 6d02017318a2461796d431962fe6a537 ({username='slave'} at 127.16.85.133:38655)
I20240502 07:58:43.646180 22998 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet b22f8e50a7d24dbdab665e47120602ce from peer 6d02017318a2461796d431962fe6a537 at {username='slave'} at 127.16.85.133:38655: session id = 6d02017318a2461796d431962fe6a537-b22f8e50a7d24dbdab665e47120602ce
I20240502 07:58:43.646615 22999 tablet_copy_source_session.cc:205] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.647053 22998 tablet_copy_source_session.cc:205] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.647425 25842 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 88c1176b415c412fb2c824f277e4d3c8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.647819 25888 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b22f8e50a7d24dbdab665e47120602ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.648015 25898 ts_tablet_manager.cc:1398] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:43.649248 25842 tablet_copy_client.cc:783] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.649367 25842 tablet_copy_client.cc:647] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.649695 25888 tablet_copy_client.cc:783] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.649785 22606 tablet_copy_service.cc:342] P 8bea0fe8d8044a2685f92f7a10c9cf85: Request end of tablet copy session 6d02017318a2461796d431962fe6a537-9e5fe86b44744df9b2e4d430fdec716f received from {username='slave'} at 127.16.85.133:57895
I20240502 07:58:43.649820 25888 tablet_copy_client.cc:647] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.649852 22606 tablet_copy_service.cc:433] P 8bea0fe8d8044a2685f92f7a10c9cf85: ending tablet copy session 6d02017318a2461796d431962fe6a537-9e5fe86b44744df9b2e4d430fdec716f on tablet 9e5fe86b44744df9b2e4d430fdec716f with peer 6d02017318a2461796d431962fe6a537
I20240502 07:58:43.653630 25842 tablet_copy_client.cc:523] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.654786 25842 tablet_bootstrap.cc:492] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:43.657720 25888 tablet_copy_client.cc:523] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.658938 25888 tablet_bootstrap.cc:492] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:43.665611 25883 tablet_bootstrap.cc:492] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.665908 25883 tablet_bootstrap.cc:492] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap complete.
I20240502 07:58:43.666165 25883 ts_tablet_manager.cc:1367] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.035s user 0.024s sys 0.003s
I20240502 07:58:43.666618 25883 raft_consensus.cc:348] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:43.666814 25883 raft_consensus.cc:729] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: LEARNER
I20240502 07:58:43.666973 25883 consensus_queue.cc:260] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:43.667352 25883 ts_tablet_manager.cc:1398] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:43.670481 25851 tablet_bootstrap.cc:492] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.670786 25851 tablet_bootstrap.cc:492] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap complete.
I20240502 07:58:43.670956 25851 ts_tablet_manager.cc:1367] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.044s user 0.023s sys 0.002s
I20240502 07:58:43.671382 25851 raft_consensus.cc:348] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:43.671487 25851 raft_consensus.cc:729] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: LEARNER
I20240502 07:58:43.671571 25851 consensus_queue.cc:260] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:43.671871 25851 ts_tablet_manager.cc:1398] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.003s sys 0.000s
I20240502 07:58:43.673619 22998 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-f1fa312fe8e44d23970c5e3ba98c2c58 received from {username='slave'} at 127.16.85.134:46615
I20240502 07:58:43.673703 22998 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-f1fa312fe8e44d23970c5e3ba98c2c58 on tablet f1fa312fe8e44d23970c5e3ba98c2c58 with peer 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:43.674293 22998 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-1d382df270004c128dc5718be32532ba received from {username='slave'} at 127.16.85.135:41731
I20240502 07:58:43.674364 22998 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-1d382df270004c128dc5718be32532ba on tablet 1d382df270004c128dc5718be32532ba with peer 4654231dfe994c1a9d2e37d9c3d75699
I20240502 07:58:43.685791 26037 ts_tablet_manager.cc:902] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:43.686285 26037 tablet_copy_client.cc:308] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:43.687116 22998 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet cfff02438656430686be0f03f5c0b8ab from peer 7eced220cf1642f3873fc385b6191e69 ({username='slave'} at 127.16.85.136:40185)
I20240502 07:58:43.687160 25842 tablet_bootstrap.cc:492] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.687196 22998 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet cfff02438656430686be0f03f5c0b8ab from peer 7eced220cf1642f3873fc385b6191e69 at {username='slave'} at 127.16.85.136:40185: session id = 7eced220cf1642f3873fc385b6191e69-cfff02438656430686be0f03f5c0b8ab
I20240502 07:58:43.687386 25842 tablet_bootstrap.cc:492] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69: Bootstrap complete.
I20240502 07:58:43.687551 25842 ts_tablet_manager.cc:1367] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.033s user 0.027s sys 0.000s
I20240502 07:58:43.687943 22998 tablet_copy_source_session.cc:205] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.687942 25842 raft_consensus.cc:348] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:43.688056 25842 raft_consensus.cc:729] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: LEARNER
I20240502 07:58:43.688174 25842 consensus_queue.cc:260] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:43.688493 25842 ts_tablet_manager.cc:1398] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 07:58:43.688896 22998 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 7eced220cf1642f3873fc385b6191e69-88c1176b415c412fb2c824f277e4d3c8 received from {username='slave'} at 127.16.85.136:40185
I20240502 07:58:43.688956 22998 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 7eced220cf1642f3873fc385b6191e69-88c1176b415c412fb2c824f277e4d3c8 on tablet 88c1176b415c412fb2c824f277e4d3c8 with peer 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:43.690274 26037 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cfff02438656430686be0f03f5c0b8ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.691882 26037 tablet_copy_client.cc:783] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.692027 26037 tablet_copy_client.cc:647] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.693013 25888 tablet_bootstrap.cc:492] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.693220 26037 tablet_copy_client.cc:523] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.693254 25888 tablet_bootstrap.cc:492] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537: Bootstrap complete.
I20240502 07:58:43.693408 25888 ts_tablet_manager.cc:1367] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.035s user 0.023s sys 0.000s
I20240502 07:58:43.693827 25888 raft_consensus.cc:348] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:43.693924 25888 raft_consensus.cc:729] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: LEARNER
I20240502 07:58:43.694042 25888 consensus_queue.cc:260] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:43.694250 26037 tablet_bootstrap.cc:492] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:43.694306 25888 ts_tablet_manager.cc:1398] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:43.694978 22998 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 6d02017318a2461796d431962fe6a537-b22f8e50a7d24dbdab665e47120602ce received from {username='slave'} at 127.16.85.133:38655
I20240502 07:58:43.695041 22998 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 6d02017318a2461796d431962fe6a537-b22f8e50a7d24dbdab665e47120602ce on tablet b22f8e50a7d24dbdab665e47120602ce with peer 6d02017318a2461796d431962fe6a537
I20240502 07:58:43.701071 25883 ts_tablet_manager.cc:902] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:43.701464 25883 tablet_copy_client.cc:308] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33387
I20240502 07:58:43.704773 22737 tablet_copy_service.cc:140] P 725969d490a04fdfb3ba890c3f5096e3: Received BeginTabletCopySession request for tablet 0e01743578f947cca1db2baec4fdee12 from peer 4654231dfe994c1a9d2e37d9c3d75699 ({username='slave'} at 127.16.85.135:38693)
I20240502 07:58:43.704870 22737 tablet_copy_service.cc:161] P 725969d490a04fdfb3ba890c3f5096e3: Beginning new tablet copy session on tablet 0e01743578f947cca1db2baec4fdee12 from peer 4654231dfe994c1a9d2e37d9c3d75699 at {username='slave'} at 127.16.85.135:38693: session id = 4654231dfe994c1a9d2e37d9c3d75699-0e01743578f947cca1db2baec4fdee12
I20240502 07:58:43.705729 22737 tablet_copy_source_session.cc:205] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:43.706646 25883 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0e01743578f947cca1db2baec4fdee12. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.708535 25883 tablet_copy_client.cc:783] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:43.708722 25883 tablet_copy_client.cc:647] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:43.715659 25883 tablet_copy_client.cc:523] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:43.716809 25883 tablet_bootstrap.cc:492] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:43.718169 26037 tablet_bootstrap.cc:492] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.718487 26037 tablet_bootstrap.cc:492] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69: Bootstrap complete.
I20240502 07:58:43.718712 26037 ts_tablet_manager.cc:1367] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.025s user 0.023s sys 0.000s
I20240502 07:58:43.719178 26037 raft_consensus.cc:348] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:43.719348 26037 raft_consensus.cc:729] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: LEARNER
I20240502 07:58:43.719468 26037 consensus_queue.cc:260] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:43.719745 26037 ts_tablet_manager.cc:1398] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:43.720187 22998 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 7eced220cf1642f3873fc385b6191e69-cfff02438656430686be0f03f5c0b8ab received from {username='slave'} at 127.16.85.136:40185
I20240502 07:58:43.720257 22998 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 7eced220cf1642f3873fc385b6191e69-cfff02438656430686be0f03f5c0b8ab on tablet cfff02438656430686be0f03f5c0b8ab with peer 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:43.721364 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:43.723297 25331 consensus_queue.cc:1035] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:43.732098 25588 tablet.cc:1722] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:43.736882 25265 raft_consensus.cc:1203] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:43.737849 25919 tablet.cc:1722] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:43.738315 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=bcb0c0b7c0d345d7ae613152d1a93ccb] alter complete (version 3)
I20240502 07:58:43.744599 25590 tablet.cc:1722] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:43.746084 25883 tablet_bootstrap.cc:492] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:43.746331 25883 tablet_bootstrap.cc:492] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap complete.
I20240502 07:58:43.746484 25883 ts_tablet_manager.cc:1367] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.030s user 0.021s sys 0.001s
I20240502 07:58:43.746860 25883 raft_consensus.cc:348] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:43.746968 25883 raft_consensus.cc:729] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: LEARNER
I20240502 07:58:43.747090 25883 consensus_queue.cc:260] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:43.747383 25883 ts_tablet_manager.cc:1398] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:43.749661 22737 tablet_copy_service.cc:342] P 725969d490a04fdfb3ba890c3f5096e3: Request end of tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-0e01743578f947cca1db2baec4fdee12 received from {username='slave'} at 127.16.85.135:38693
I20240502 07:58:43.749745 22737 tablet_copy_service.cc:433] P 725969d490a04fdfb3ba890c3f5096e3: ending tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-0e01743578f947cca1db2baec4fdee12 on tablet 0e01743578f947cca1db2baec4fdee12 with peer 4654231dfe994c1a9d2e37d9c3d75699
I20240502 07:58:43.754132 25331 consensus_queue.cc:1035] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:43.809056 22957 tablet_service.cc:1429] Tablet server has 19 leaders and 0 scanners
I20240502 07:58:43.811571 22827 tablet_service.cc:1429] Tablet server has 12 leaders and 0 scanners
I20240502 07:58:43.813482 24984 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:43.813730 24851 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:43.816404 22566 tablet_service.cc:1429] Tablet server has 14 leaders and 0 scanners
I20240502 07:58:43.817165 25114 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:43.827989 22695 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 07:58:43.832448 25004 raft_consensus.cc:1203] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:43.848902 22438 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:43.849066 22438 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:43.851188 22438 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 4b3baeb1420c4178addd68ce3e1d9606
I20240502 07:58:43.851941 24984 tablet_service.cc:1514] Processing DeleteTablet for tablet 4b3baeb1420c4178addd68ce3e1d9606 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:42878
I20240502 07:58:43.851918 24851 tablet_service.cc:1514] Processing DeleteTablet for tablet 4b3baeb1420c4178addd68ce3e1d9606 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:43.852113 25823 tablet_replica.cc:330] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e: stopping tablet replica
I20240502 07:58:43.852161 25822 tablet_replica.cc:330] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:43.852241 25823 raft_consensus.cc:2229] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:43.852197 25114 tablet_service.cc:1514] Processing DeleteTablet for tablet 4b3baeb1420c4178addd68ce3e1d9606 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:43.852284 25822 raft_consensus.cc:2229] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:43.852386 25822 raft_consensus.cc:2258] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.852411 25823 raft_consensus.cc:2258] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.853250 25823 ts_tablet_manager.cc:1875] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.853338 25822 ts_tablet_manager.cc:1875] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.855116 25823 ts_tablet_manager.cc:1888] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:43.855192 25823 log.cc:1196] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/wals/4b3baeb1420c4178addd68ce3e1d9606
I20240502 07:58:43.855341 25822 ts_tablet_manager.cc:1888] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:43.855407 25822 log.cc:1196] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/4b3baeb1420c4178addd68ce3e1d9606
I20240502 07:58:43.855441 25823 ts_tablet_manager.cc:1909] T 4b3baeb1420c4178addd68ce3e1d9606 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting consensus metadata
I20240502 07:58:43.855669 25822 ts_tablet_manager.cc:1909] T 4b3baeb1420c4178addd68ce3e1d9606 P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:43.856072 22430 catalog_manager.cc:4830] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): tablet 4b3baeb1420c4178addd68ce3e1d9606 (table rebalancer_dynamic_table_new_name [id=bcb0c0b7c0d345d7ae613152d1a93ccb]) successfully deleted
I20240502 07:58:43.856209 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet 4b3baeb1420c4178addd68ce3e1d9606 (table rebalancer_dynamic_table_new_name [id=bcb0c0b7c0d345d7ae613152d1a93ccb]) successfully deleted
I20240502 07:58:43.859318 25235 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:43.860936 26093 tablet_replica.cc:330] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:43.861317 26093 raft_consensus.cc:2229] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:43.861546 26093 raft_consensus.cc:2258] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.863879 26093 ts_tablet_manager.cc:1875] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.865684 26093 ts_tablet_manager.cc:1888] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:43.865757 26093 log.cc:1196] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/4b3baeb1420c4178addd68ce3e1d9606
I20240502 07:58:43.866032 26093 ts_tablet_manager.cc:1909] T 4b3baeb1420c4178addd68ce3e1d9606 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:43.867056 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet 4b3baeb1420c4178addd68ce3e1d9606 (table rebalancer_dynamic_table_new_name [id=bcb0c0b7c0d345d7ae613152d1a93ccb]) successfully deleted
I20240502 07:58:43.873734 25492 consensus_queue.cc:1035] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:43.876396 25874 consensus_queue.cc:1035] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:43.897183 25004 raft_consensus.cc:1203] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:43.899610 25265 raft_consensus.cc:1203] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:43.911931 22845 consensus_queue.cc:237] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:43.913276 22584 raft_consensus.cc:1261] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:43.913534 22783 consensus_peers.cc:485] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 -> Peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Couldn't send request to peer 7eced220cf1642f3873fc385b6191e69. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 45499d82e94c47b8b398c159df9fd49b. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:43.913777 22979 raft_consensus.cc:1261] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.914075 26079 consensus_queue.cc:1035] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.914371 26079 consensus_queue.cc:1035] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.915971 25532 raft_consensus.cc:2932] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:43.916386 22584 raft_consensus.cc:2932] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:43.918174 22979 raft_consensus.cc:2932] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:43.919207 22444 catalog_manager.cc:5484] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:43.920861 25134 raft_consensus.cc:1203] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:43.921669 24871 raft_consensus.cc:1203] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:43.923405 22444 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_6" } modify_external_catalogs: true
I20240502 07:58:43.923516 22444 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_6" } modify_external_catalogs: true
I20240502 07:58:43.925832 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 1a5ce428e35849f7a3f2148f02fede71
I20240502 07:58:43.926105 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 4 replicas of tablet ddebb5e20e134841886be53f8f0d5e02
I20240502 07:58:43.926340 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 5c36c7654df64dbebdf72ddf403771ab
I20240502 07:58:43.926496 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 1a5ce428e35849f7a3f2148f02fede71 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:43.926653 25468 tablet_replica.cc:330] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:43.926779 25468 raft_consensus.cc:2229] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:43.926890 25468 raft_consensus.cc:2258] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.927063 22564 tablet_service.cc:1514] Processing DeleteTablet for tablet 5c36c7654df64dbebdf72ddf403771ab with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:43.927117 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet ddebb5e20e134841886be53f8f0d5e02 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:43.927382 25468 ts_tablet_manager.cc:1875] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.927909 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet 5c36c7654df64dbebdf72ddf403771ab with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:43.928058 25465 tablet_replica.cc:330] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:43.928174 25465 raft_consensus.cc:2229] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:43.928304 25465 raft_consensus.cc:2258] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.928764 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet ddebb5e20e134841886be53f8f0d5e02 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:43.928874 25465 ts_tablet_manager.cc:1875] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.929116 22958 tablet_service.cc:1514] Processing DeleteTablet for tablet ddebb5e20e134841886be53f8f0d5e02 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:43.929463 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet 1a5ce428e35849f7a3f2148f02fede71 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:43.929546 26109 tablet_replica.cc:330] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:43.929620 25467 tablet_replica.cc:330] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:43.929690 26109 raft_consensus.cc:2229] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:43.929755 25467 raft_consensus.cc:2229] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:43.929822 26109 raft_consensus.cc:2258] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.929880 25467 raft_consensus.cc:2258] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.930325 25467 ts_tablet_manager.cc:1875] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.930343 26109 ts_tablet_manager.cc:1875] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.931071 25465 ts_tablet_manager.cc:1888] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:43.931131 25465 log.cc:1196] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/5c36c7654df64dbebdf72ddf403771ab
I20240502 07:58:43.931391 25465 ts_tablet_manager.cc:1909] T 5c36c7654df64dbebdf72ddf403771ab P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:43.931983 25465 tablet_replica.cc:330] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:43.932042 25467 ts_tablet_manager.cc:1888] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:43.932080 25465 raft_consensus.cc:2229] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:43.932104 25467 log.cc:1196] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/ddebb5e20e134841886be53f8f0d5e02
I20240502 07:58:43.932124 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 5c36c7654df64dbebdf72ddf403771ab (table rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]) successfully deleted
I20240502 07:58:43.932173 25465 raft_consensus.cc:2258] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.932340 25467 ts_tablet_manager.cc:1909] T ddebb5e20e134841886be53f8f0d5e02 P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:43.932574 25465 ts_tablet_manager.cc:1875] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.932785 26109 ts_tablet_manager.cc:1888] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:43.932817 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet 1a5ce428e35849f7a3f2148f02fede71 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:43.932854 26109 log.cc:1196] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/ddebb5e20e134841886be53f8f0d5e02
I20240502 07:58:43.932947 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet 5c36c7654df64dbebdf72ddf403771ab with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:43.933126 26109 ts_tablet_manager.cc:1909] T ddebb5e20e134841886be53f8f0d5e02 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:43.933297 24851 tablet_service.cc:1514] Processing DeleteTablet for tablet ddebb5e20e134841886be53f8f0d5e02 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:43 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:43.933487 25822 tablet_replica.cc:330] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:43.933617 25822 raft_consensus.cc:2229] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Raft consensus shutting down.
I20240502 07:58:43.933709 25822 raft_consensus.cc:2258] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Raft consensus is shut down!
I20240502 07:58:43.934818 25134 raft_consensus.cc:1203] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:43.935161 25465 ts_tablet_manager.cc:1888] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:43.935230 25465 log.cc:1196] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/1a5ce428e35849f7a3f2148f02fede71
I20240502 07:58:43.935217 25822 ts_tablet_manager.cc:1875] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.935626 25465 ts_tablet_manager.cc:1909] T 1a5ce428e35849f7a3f2148f02fede71 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:43.935844 26109 tablet_replica.cc:330] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:43.935961 26109 raft_consensus.cc:2229] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:43.936117 26109 raft_consensus.cc:2258] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.936206 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 1a5ce428e35849f7a3f2148f02fede71 (table rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]) successfully deleted
I20240502 07:58:43.936573 26109 ts_tablet_manager.cc:1875] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.937101 25822 ts_tablet_manager.cc:1888] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:43.937170 25822 log.cc:1196] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/ddebb5e20e134841886be53f8f0d5e02
I20240502 07:58:43.937461 25822 ts_tablet_manager.cc:1909] T ddebb5e20e134841886be53f8f0d5e02 P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:43.937987 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet ddebb5e20e134841886be53f8f0d5e02 (table rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]) successfully deleted
I20240502 07:58:43.938242 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet ddebb5e20e134841886be53f8f0d5e02 (table rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]) successfully deleted
I20240502 07:58:43.939276 26109 ts_tablet_manager.cc:1888] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:43.939277 25468 ts_tablet_manager.cc:1888] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:43.939342 25468 log.cc:1196] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/1a5ce428e35849f7a3f2148f02fede71
I20240502 07:58:43.939342 26109 log.cc:1196] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/1a5ce428e35849f7a3f2148f02fede71
I20240502 07:58:43.939607 26109 ts_tablet_manager.cc:1909] T 1a5ce428e35849f7a3f2148f02fede71 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:43.939607 25468 ts_tablet_manager.cc:1909] T 1a5ce428e35849f7a3f2148f02fede71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:43.940136 25468 tablet_replica.cc:330] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:43.940135 26109 tablet_replica.cc:330] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:43.940223 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 1a5ce428e35849f7a3f2148f02fede71 (table rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]) successfully deleted
I20240502 07:58:43.940248 26109 raft_consensus.cc:2229] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:43.940317 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 1a5ce428e35849f7a3f2148f02fede71 (table rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]) successfully deleted
I20240502 07:58:43.940371 26109 raft_consensus.cc:2258] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.940760 26109 ts_tablet_manager.cc:1875] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.940917 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet ddebb5e20e134841886be53f8f0d5e02 (table rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]) successfully deleted
I20240502 07:58:43.941644 25468 raft_consensus.cc:2229] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:43.941790 25468 raft_consensus.cc:2258] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.942245 25468 ts_tablet_manager.cc:1875] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.942452 26109 ts_tablet_manager.cc:1888] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:43.942523 26109 log.cc:1196] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/5c36c7654df64dbebdf72ddf403771ab
I20240502 07:58:43.942765 26109 ts_tablet_manager.cc:1909] T 5c36c7654df64dbebdf72ddf403771ab P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:43.943420 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 5c36c7654df64dbebdf72ddf403771ab (table rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]) successfully deleted
I20240502 07:58:43.944008 25468 ts_tablet_manager.cc:1888] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:43.944088 25468 log.cc:1196] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/5c36c7654df64dbebdf72ddf403771ab
I20240502 07:58:43.944272 25468 ts_tablet_manager.cc:1909] T 5c36c7654df64dbebdf72ddf403771ab P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:43.944768 25468 tablet_replica.cc:330] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:43.944821 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 5c36c7654df64dbebdf72ddf403771ab (table rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]) successfully deleted
I20240502 07:58:43.944905 25468 raft_consensus.cc:2229] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:43.945093 25468 raft_consensus.cc:2258] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:43.945425 25468 ts_tablet_manager.cc:1875] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:43.946847 25468 ts_tablet_manager.cc:1888] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:43.946942 25468 log.cc:1196] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/ddebb5e20e134841886be53f8f0d5e02
I20240502 07:58:43.947157 25468 ts_tablet_manager.cc:1909] T ddebb5e20e134841886be53f8f0d5e02 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:43.947697 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet ddebb5e20e134841886be53f8f0d5e02 (table rebalance_test_table_6 [id=17bc829bde1c41e0a6eafa330aa392d1]) successfully deleted
I20240502 07:58:43.950785 22979 consensus_queue.cc:237] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:43.951957 22845 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.952034 22715 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:43.952113 22912 consensus_peers.cc:485] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): Couldn't send request to peer 6d02017318a2461796d431962fe6a537. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9d2283e5d28347519b38e876d840608e. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:43.952378 25670 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.952556 25495 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.953706 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:43.954562 25495 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:43.954921 22845 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:43.957108 22715 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:43.959317 22436 catalog_manager.cc:5484] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:43.960083 25235 tablet_service.cc:1467] Processing CreateTablet for tablet efadbab41bd24244bcc0247234ea27a3 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=c403b21f2dee470394eb271267f2a442]), partition=
I20240502 07:58:43.960245 22827 tablet_service.cc:1467] Processing CreateTablet for tablet efadbab41bd24244bcc0247234ea27a3 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=c403b21f2dee470394eb271267f2a442]), partition=
I20240502 07:58:43.960317 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet efadbab41bd24244bcc0247234ea27a3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.960467 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet efadbab41bd24244bcc0247234ea27a3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.961068 25114 tablet_service.cc:1467] Processing CreateTablet for tablet efadbab41bd24244bcc0247234ea27a3 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=c403b21f2dee470394eb271267f2a442]), partition=
I20240502 07:58:43.961256 25114 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet efadbab41bd24244bcc0247234ea27a3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:43.962576 26120 tablet_bootstrap.cc:492] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:43.963639 26120 tablet_bootstrap.cc:654] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:43.964637 26120 tablet_bootstrap.cc:492] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:43.964939 26121 tablet_bootstrap.cc:492] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:43.965001 26120 ts_tablet_manager.cc:1367] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:43.965409 26120 raft_consensus.cc:348] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:43.965493 26120 raft_consensus.cc:374] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:43.965518 26120 raft_consensus.cc:729] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:43.965649 26120 consensus_queue.cc:260] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:43.965901 26120 ts_tablet_manager.cc:1398] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:43.966272 26121 tablet_bootstrap.cc:654] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:43.967139 26119 tablet_bootstrap.cc:492] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:43.967620 25134 raft_consensus.cc:1203] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:43.969040 26119 tablet_bootstrap.cc:654] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:43.971117 26119 tablet_bootstrap.cc:492] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:43.971204 26119 ts_tablet_manager.cc:1367] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:43.971346 26121 tablet_bootstrap.cc:492] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:43.971536 26121 ts_tablet_manager.cc:1367] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
I20240502 07:58:43.971662 26119 raft_consensus.cc:348] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:43.971774 26119 raft_consensus.cc:374] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:43.971812 26119 raft_consensus.cc:729] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:43.971936 26119 consensus_queue.cc:260] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:43.972020 26121 raft_consensus.cc:348] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:43.972102 26121 raft_consensus.cc:374] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:43.972137 26121 raft_consensus.cc:729] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:43.972198 26119 ts_tablet_manager.cc:1398] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:43.972244 26121 consensus_queue.cc:260] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:43.972996 26121 ts_tablet_manager.cc:1398] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:43.973193 22979 consensus_queue.cc:237] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
W20240502 07:58:43.974853 22914 consensus_peers.cc:485] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1e370c8fe68b4392ba3720533e538de3. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:43.975426 22845 raft_consensus.cc:1261] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.975747 22584 raft_consensus.cc:1261] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:43.976054 25669 consensus_queue.cc:1035] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.976239 25670 consensus_queue.cc:1035] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:43.977788 25670 raft_consensus.cc:2932] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:43.978080 22584 raft_consensus.cc:2932] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:43.979689 22845 raft_consensus.cc:2932] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:43.981047 22440 catalog_manager.cc:5484] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:43.988390 24871 raft_consensus.cc:1203] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:44.000231 25004 raft_consensus.cc:1203] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:44.013177 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_4"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:44.019212 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 415879c2eee74932b9ac45282f6dd0fb (DEFAULT_TABLE table=rebalancer_extra_table_4 [id=61a72ea9962740bd8839987228b47853]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.019470 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 415879c2eee74932b9ac45282f6dd0fb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.019644 25235 tablet_service.cc:1467] Processing CreateTablet for tablet c3a7b2e52ba641e993cf1d5dc9724473 (DEFAULT_TABLE table=rebalancer_extra_table_4 [id=61a72ea9962740bd8839987228b47853]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.019856 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c3a7b2e52ba641e993cf1d5dc9724473. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.019809 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 855e40f5f05948eb8902d0a6771fc0bf (DEFAULT_TABLE table=rebalancer_extra_table_4 [id=61a72ea9962740bd8839987228b47853]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.019973 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 855e40f5f05948eb8902d0a6771fc0bf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.019951 22695 tablet_service.cc:1467] Processing CreateTablet for tablet c3a7b2e52ba641e993cf1d5dc9724473 (DEFAULT_TABLE table=rebalancer_extra_table_4 [id=61a72ea9962740bd8839987228b47853]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.020160 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c3a7b2e52ba641e993cf1d5dc9724473. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.022068 26119 tablet_bootstrap.cc:492] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:44.022430 24851 tablet_service.cc:1467] Processing CreateTablet for tablet 415879c2eee74932b9ac45282f6dd0fb (DEFAULT_TABLE table=rebalancer_extra_table_4 [id=61a72ea9962740bd8839987228b47853]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.022624 24851 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 415879c2eee74932b9ac45282f6dd0fb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.022982 25629 consensus_queue.cc:1035] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:44.023157 26119 tablet_bootstrap.cc:654] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.024820 24852 tablet_service.cc:1467] Processing CreateTablet for tablet c3a7b2e52ba641e993cf1d5dc9724473 (DEFAULT_TABLE table=rebalancer_extra_table_4 [id=61a72ea9962740bd8839987228b47853]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.025010 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c3a7b2e52ba641e993cf1d5dc9724473. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.025012 26119 tablet_bootstrap.cc:492] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:44.025079 26119 ts_tablet_manager.cc:1367] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:44.025467 26119 raft_consensus.cc:348] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.025553 26119 raft_consensus.cc:374] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.025578 26119 raft_consensus.cc:729] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.025688 26119 consensus_queue.cc:260] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.025959 26119 ts_tablet_manager.cc:1398] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:44.026104 26119 tablet_bootstrap.cc:492] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:44.026741 26119 tablet_bootstrap.cc:654] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.026831 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 855e40f5f05948eb8902d0a6771fc0bf (DEFAULT_TABLE table=rebalancer_extra_table_4 [id=61a72ea9962740bd8839987228b47853]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.027021 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 855e40f5f05948eb8902d0a6771fc0bf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.027603 26119 tablet_bootstrap.cc:492] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:44.027667 26119 ts_tablet_manager.cc:1367] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:44.028041 26119 raft_consensus.cc:348] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.028118 26119 raft_consensus.cc:374] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.028143 26119 raft_consensus.cc:729] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.028234 26119 consensus_queue.cc:260] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.028453 26119 ts_tablet_manager.cc:1398] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:44.028740 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 415879c2eee74932b9ac45282f6dd0fb (DEFAULT_TABLE table=rebalancer_extra_table_4 [id=61a72ea9962740bd8839987228b47853]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.028971 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 415879c2eee74932b9ac45282f6dd0fb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.029596 25004 raft_consensus.cc:3037] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.031246 26178 tablet_bootstrap.cc:492] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:44.031282 25629 consensus_queue.cc:1035] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:44.032147 25114 tablet_service.cc:1467] Processing CreateTablet for tablet 855e40f5f05948eb8902d0a6771fc0bf (DEFAULT_TABLE table=rebalancer_extra_table_4 [id=61a72ea9962740bd8839987228b47853]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.032413 25114 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 855e40f5f05948eb8902d0a6771fc0bf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.032743 26178 tablet_bootstrap.cc:654] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.033403 26172 tablet_bootstrap.cc:492] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:44.034348 26172 tablet_bootstrap.cc:654] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.034549 26175 tablet_bootstrap.cc:492] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:44.034690 26178 tablet_bootstrap.cc:492] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:44.034782 26178 ts_tablet_manager.cc:1367] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.001s
I20240502 07:58:44.035143 26178 raft_consensus.cc:348] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:44.035235 26178 raft_consensus.cc:374] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.035269 26178 raft_consensus.cc:729] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.035380 26178 consensus_queue.cc:260] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:44.035650 26178 ts_tablet_manager.cc:1398] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:44.035722 26171 tablet_bootstrap.cc:492] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:44.035778 26121 tablet_bootstrap.cc:492] T 855e40f5f05948eb8902d0a6771fc0bf P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:44.036468 26121 tablet_bootstrap.cc:654] T 855e40f5f05948eb8902d0a6771fc0bf P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.036479 26171 tablet_bootstrap.cc:654] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.035485 26175 tablet_bootstrap.cc:654] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.037410 26121 tablet_bootstrap.cc:492] T 855e40f5f05948eb8902d0a6771fc0bf P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:44.037485 26121 ts_tablet_manager.cc:1367] T 855e40f5f05948eb8902d0a6771fc0bf P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:44.037859 26121 raft_consensus.cc:348] T 855e40f5f05948eb8902d0a6771fc0bf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.037959 26121 raft_consensus.cc:374] T 855e40f5f05948eb8902d0a6771fc0bf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.037987 26121 raft_consensus.cc:729] T 855e40f5f05948eb8902d0a6771fc0bf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.038108 26121 consensus_queue.cc:260] T 855e40f5f05948eb8902d0a6771fc0bf P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.038349 26121 ts_tablet_manager.cc:1398] T 855e40f5f05948eb8902d0a6771fc0bf P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:44.038393 26175 tablet_bootstrap.cc:492] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:44.038452 26175 ts_tablet_manager.cc:1367] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:44.038755 26175 raft_consensus.cc:348] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:44.038848 26175 raft_consensus.cc:374] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.038884 26175 raft_consensus.cc:729] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.038990 26175 consensus_queue.cc:260] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:44.039181 26172 tablet_bootstrap.cc:492] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:44.039193 26175 ts_tablet_manager.cc:1398] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:44.039319 26175 tablet_bootstrap.cc:492] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:44.039335 26172 ts_tablet_manager.cc:1367] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:58:44.039711 26171 tablet_bootstrap.cc:492] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:44.039778 26171 ts_tablet_manager.cc:1367] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:44.039723 26172 raft_consensus.cc:348] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.039819 26172 raft_consensus.cc:374] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.039851 26172 raft_consensus.cc:729] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.039940 26175 tablet_bootstrap.cc:654] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.039955 26172 consensus_queue.cc:260] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.040205 26172 ts_tablet_manager.cc:1398] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:44.040158 26171 raft_consensus.cc:348] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:44.040247 26171 raft_consensus.cc:374] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.041061 26171 raft_consensus.cc:729] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.041178 26171 consensus_queue.cc:260] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:44.041411 26171 ts_tablet_manager.cc:1398] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:44.045261 26175 tablet_bootstrap.cc:492] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:44.045351 26175 ts_tablet_manager.cc:1367] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:58:44.045780 26175 raft_consensus.cc:348] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.045866 26175 raft_consensus.cc:374] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.045908 26175 raft_consensus.cc:729] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.046007 26175 consensus_queue.cc:260] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.046262 26175 ts_tablet_manager.cc:1398] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:44.046409 26175 tablet_bootstrap.cc:492] T 855e40f5f05948eb8902d0a6771fc0bf P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:44.047202 26175 tablet_bootstrap.cc:654] T 855e40f5f05948eb8902d0a6771fc0bf P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.048346 26175 tablet_bootstrap.cc:492] T 855e40f5f05948eb8902d0a6771fc0bf P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:44.048753 26175 ts_tablet_manager.cc:1367] T 855e40f5f05948eb8902d0a6771fc0bf P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:44.049104 26175 raft_consensus.cc:348] T 855e40f5f05948eb8902d0a6771fc0bf P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.049185 26175 raft_consensus.cc:374] T 855e40f5f05948eb8902d0a6771fc0bf P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.049214 26175 raft_consensus.cc:729] T 855e40f5f05948eb8902d0a6771fc0bf P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.049317 26175 consensus_queue.cc:260] T 855e40f5f05948eb8902d0a6771fc0bf P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.049885 26175 ts_tablet_manager.cc:1398] T 855e40f5f05948eb8902d0a6771fc0bf P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:44.060434 25134 raft_consensus.cc:1203] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:44.068156 25987 raft_consensus.cc:482] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:44.068255 25987 raft_consensus.cc:504] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:44.068603 25987 leader_election.cc:288] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:44.068893 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "415879c2eee74932b9ac45282f6dd0fb" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:44.069023 24871 raft_consensus.cc:2454] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:44.069166 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "415879c2eee74932b9ac45282f6dd0fb" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:44.069283 22979 raft_consensus.cc:2454] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:44.069304 22519 leader_election.cc:302] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:44.069500 25987 raft_consensus.cc:2781] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:44.069559 25987 raft_consensus.cc:482] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:44.069599 25987 raft_consensus.cc:3037] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.070421 25987 raft_consensus.cc:504] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:44.070751 25987 leader_election.cc:288] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:44.071034 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "415879c2eee74932b9ac45282f6dd0fb" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:44.071143 24871 raft_consensus.cc:3037] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.071125 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "415879c2eee74932b9ac45282f6dd0fb" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:44.071228 22979 raft_consensus.cc:3037] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.072046 22979 raft_consensus.cc:2454] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:44.072046 24871 raft_consensus.cc:2454] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:44.072268 22519 leader_election.cc:302] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:44.072397 25987 raft_consensus.cc:2781] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:44.072480 25987 raft_consensus.cc:686] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:44.072605 25987 consensus_queue.cc:237] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:44.074647 22444 catalog_manager.cc:5484] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:44.074872 25265 raft_consensus.cc:1203] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:44.086356 25350 raft_consensus.cc:482] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:44.086481 25350 raft_consensus.cc:504] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.086850 25350 leader_election.cc:288] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:44.087118 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "855e40f5f05948eb8902d0a6771fc0bf" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:44.087232 24871 raft_consensus.cc:2454] T 855e40f5f05948eb8902d0a6771fc0bf P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:44.087355 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "855e40f5f05948eb8902d0a6771fc0bf" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:44.087483 25199 leader_election.cc:302] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:44.087502 25134 raft_consensus.cc:2454] T 855e40f5f05948eb8902d0a6771fc0bf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:44.087620 25350 raft_consensus.cc:2781] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:44.087663 25350 raft_consensus.cc:482] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:44.087709 25350 raft_consensus.cc:3037] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.088738 25350 raft_consensus.cc:504] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.089097 25350 leader_election.cc:288] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:44.089514 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "855e40f5f05948eb8902d0a6771fc0bf" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:44.089601 24871 raft_consensus.cc:3037] T 855e40f5f05948eb8902d0a6771fc0bf P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.090224 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "855e40f5f05948eb8902d0a6771fc0bf" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:44.090343 25134 raft_consensus.cc:3037] T 855e40f5f05948eb8902d0a6771fc0bf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.090484 24871 raft_consensus.cc:2454] T 855e40f5f05948eb8902d0a6771fc0bf P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:44.090785 25199 leader_election.cc:302] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:44.091007 25350 raft_consensus.cc:2781] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:44.091096 25350 raft_consensus.cc:686] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:44.091248 25350 consensus_queue.cc:237] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.091293 25134 raft_consensus.cc:2454] T 855e40f5f05948eb8902d0a6771fc0bf P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:44.093230 22440 catalog_manager.cc:5484] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:44.126999 25331 raft_consensus.cc:482] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:44.127112 25331 raft_consensus.cc:504] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:44.127487 25331 leader_election.cc:288] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:44.127792 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "efadbab41bd24244bcc0247234ea27a3" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:44.127864 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "efadbab41bd24244bcc0247234ea27a3" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:44.127925 25134 raft_consensus.cc:2454] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:44.127976 22847 raft_consensus.cc:2454] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:44.128178 25202 leader_election.cc:302] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:44.128508 25331 raft_consensus.cc:2781] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:44.128568 25331 raft_consensus.cc:482] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:44.128603 25331 raft_consensus.cc:3037] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.129567 25331 raft_consensus.cc:504] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:44.129848 25331 leader_election.cc:288] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:44.130244 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "efadbab41bd24244bcc0247234ea27a3" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:44.130275 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "efadbab41bd24244bcc0247234ea27a3" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:44.130379 25134 raft_consensus.cc:3037] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.130379 22845 raft_consensus.cc:3037] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.131327 25134 raft_consensus.cc:2454] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:44.131388 22845 raft_consensus.cc:2454] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:44.131654 25202 leader_election.cc:302] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:44.131795 25331 raft_consensus.cc:2781] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:44.131901 25331 raft_consensus.cc:686] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:44.132028 25331 consensus_queue.cc:237] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:44.133644 22444 catalog_manager.cc:5484] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:44.172739 25004 raft_consensus.cc:1203] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:44.190943 26079 raft_consensus.cc:1050] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88: attempting to promote NON_VOTER 7eced220cf1642f3873fc385b6191e69 to VOTER
I20240502 07:58:44.191375 26079 consensus_queue.cc:237] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:44.192638 25265 raft_consensus.cc:1261] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.192909 22715 raft_consensus.cc:1261] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.193099 26079 consensus_queue.cc:1035] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.193464 22584 raft_consensus.cc:1261] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.193595 26079 consensus_queue.cc:1035] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.193904 26079 consensus_queue.cc:1035] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.195466 25532 raft_consensus.cc:2932] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.195901 22715 raft_consensus.cc:2932] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.197101 22584 raft_consensus.cc:2932] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.198465 26150 raft_consensus.cc:482] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:44.198556 26150 raft_consensus.cc:504] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.198974 26150 leader_election.cc:288] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:44.199054 22440 catalog_manager.cc:5484] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:44.202329 25265 raft_consensus.cc:2932] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.203330 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3a7b2e52ba641e993cf1d5dc9724473" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:44.203464 25266 raft_consensus.cc:2454] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:44.203783 22652 leader_election.cc:302] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:44.204116 26150 raft_consensus.cc:2781] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:44.204178 26150 raft_consensus.cc:482] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:44.204209 26150 raft_consensus.cc:3037] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.205109 26150 raft_consensus.cc:504] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.205417 26150 leader_election.cc:288] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:44.205724 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3a7b2e52ba641e993cf1d5dc9724473" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:44.205703 22847 consensus_queue.cc:237] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:44.205837 25265 raft_consensus.cc:3037] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.207903 25266 raft_consensus.cc:1261] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.208317 22715 raft_consensus.cc:1261] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.208294 26284 consensus_queue.cc:1035] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.209781 26284 consensus_queue.cc:1035] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.210047 25265 raft_consensus.cc:2454] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:44.210366 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3a7b2e52ba641e993cf1d5dc9724473" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:44.210372 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c3a7b2e52ba641e993cf1d5dc9724473" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:44.210533 24871 raft_consensus.cc:2454] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:44.210575 26284 raft_consensus.cc:2932] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.210697 22652 leader_election.cc:302] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:44.210872 26150 raft_consensus.cc:2781] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:44.210951 26150 raft_consensus.cc:686] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:44.211066 26150 consensus_queue.cc:237] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.212472 25265 raft_consensus.cc:2932] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.212734 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 90480cc47cac4c84994bfcb9bc8a2dbf with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:44.212849 22444 catalog_manager.cc:5484] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:44.214802 22441 catalog_manager.cc:5484] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:44.215363 22715 raft_consensus.cc:2932] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.216816 25265 raft_consensus.cc:1203] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:44.219610 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 90480cc47cac4c84994bfcb9bc8a2dbf with delete_type TABLET_DATA_TOMBSTONED (TS 8bea0fe8d8044a2685f92f7a10c9cf85 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:44.220005 25468 tablet_replica.cc:330] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:44.220151 25468 raft_consensus.cc:2229] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.220312 25468 raft_consensus.cc:2258] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.221320 25468 ts_tablet_manager.cc:1875] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:44.223070 25468 ts_tablet_manager.cc:1888] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:44.223119 25468 log.cc:1196] T 90480cc47cac4c84994bfcb9bc8a2dbf P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/90480cc47cac4c84994bfcb9bc8a2dbf
I20240502 07:58:44.223537 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 90480cc47cac4c84994bfcb9bc8a2dbf (table rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]) successfully deleted
I20240502 07:58:44.233114 24871 raft_consensus.cc:1203] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:44.234598 25114 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:44.236922 24850 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:44.238251 26128 raft_consensus.cc:1050] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e to VOTER
I20240502 07:58:44.238279 22695 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 07:58:44.238807 26128 consensus_queue.cc:237] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:44.240028 24984 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:44.241564 25004 raft_consensus.cc:1261] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.241869 22847 raft_consensus.cc:1261] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.241988 25495 consensus_queue.cc:1035] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.242272 25669 consensus_queue.cc:1035] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.245191 22715 raft_consensus.cc:1261] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.245527 25669 consensus_queue.cc:1035] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.251986 22959 tablet_service.cc:1429] Tablet server has 17 leaders and 0 scanners
I20240502 07:58:44.252797 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:44.268698 25669 raft_consensus.cc:2932] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.268848 25246 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 07:58:44.269323 25134 raft_consensus.cc:1203] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:44.270671 22715 raft_consensus.cc:2932] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.270742 25004 raft_consensus.cc:2932] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.273629 22440 catalog_manager.cc:5484] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:44.279546 22847 raft_consensus.cc:2932] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.279871 22979 consensus_queue.cc:237] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:44.281566 22715 raft_consensus.cc:1261] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.281872 25004 raft_consensus.cc:1261] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.282207 25495 consensus_queue.cc:1035] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.285117 25670 consensus_queue.cc:1035] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.285585 22827 tablet_service.cc:1429] Tablet server has 12 leaders and 0 scanners
I20240502 07:58:44.286556 25670 raft_consensus.cc:2932] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.286983 22715 raft_consensus.cc:2932] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.287865 22429 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a8dcc36740be4ca3ac1d43591f7549bd with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:44.289454 22440 catalog_manager.cc:5484] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.293776 22847 raft_consensus.cc:1261] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:44.294090 25134 raft_consensus.cc:1261] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:44.294420 25350 consensus_queue.cc:1035] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:44.294539 26262 consensus_queue.cc:1035] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:44.294708 22566 tablet_service.cc:1429] Tablet server has 14 leaders and 0 scanners
I20240502 07:58:44.295814 26134 raft_consensus.cc:1050] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85: attempting to promote NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 to VOTER
I20240502 07:58:44.295806 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet a8dcc36740be4ca3ac1d43591f7549bd with delete_type TABLET_DATA_TOMBSTONED (TS 7db4178eed174225bfb24f5f9445ce88 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:44.295985 25467 tablet_replica.cc:330] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:44.296103 25467 raft_consensus.cc:2229] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.296234 25467 raft_consensus.cc:2258] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.296252 26134 consensus_queue.cc:237] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:44.296911 25467 ts_tablet_manager.cc:1875] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:44.297612 25135 raft_consensus.cc:1261] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.297622 22715 raft_consensus.cc:1261] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.297947 25987 consensus_queue.cc:1035] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.298235 25987 consensus_queue.cc:1035] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.298790 25467 ts_tablet_manager.cc:1888] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:44.298856 25467 log.cc:1196] T a8dcc36740be4ca3ac1d43591f7549bd P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/a8dcc36740be4ca3ac1d43591f7549bd
I20240502 07:58:44.299750 22442 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet efadbab41bd24244bcc0247234ea27a3 (table rebalancer_dynamic_table [id=c403b21f2dee470394eb271267f2a442]). Expected version 1 got 0
I20240502 07:58:44.300220 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet a8dcc36740be4ca3ac1d43591f7549bd (table rebalance_test_table_19 [id=65deae5d69b44af784a28c39cd113241]) successfully deleted
I20240502 07:58:44.300300 22845 raft_consensus.cc:1261] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.300567 25004 raft_consensus.cc:2932] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.300758 22440 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet efadbab41bd24244bcc0247234ea27a3 (table rebalancer_dynamic_table [id=c403b21f2dee470394eb271267f2a442]). Expected version 1 got 0
I20240502 07:58:44.303248 26314 tablet.cc:1722] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:44.303679 26314 tablet.cc:1620] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:44.304219 26315 tablet.cc:1722] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:44.304355 22440 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet efadbab41bd24244bcc0247234ea27a3 (table rebalancer_dynamic_table [id=c403b21f2dee470394eb271267f2a442]). Expected version 1 got 0
I20240502 07:58:44.304682 26315 tablet.cc:1620] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:44.305866 26134 consensus_queue.cc:1035] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.307441 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=c403b21f2dee470394eb271267f2a442] alter complete (version 1)
I20240502 07:58:44.310895 22440 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet efadbab41bd24244bcc0247234ea27a3 (table rebalancer_dynamic_table [id=c403b21f2dee470394eb271267f2a442]). Expected version 1 got 0
I20240502 07:58:44.312139 25987 raft_consensus.cc:2932] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.313877 22715 raft_consensus.cc:2932] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.313859 25135 raft_consensus.cc:2932] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.314822 22847 raft_consensus.cc:2932] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.315732 22440 catalog_manager.cc:5484] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:44.318306 22584 consensus_queue.cc:237] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:44.321857 26317 tablet.cc:1722] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:44.322064 26317 tablet.cc:1620] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:58:44.323468 22847 raft_consensus.cc:1261] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.323820 26311 consensus_queue.cc:1035] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.325968 25134 raft_consensus.cc:1261] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.326457 26311 consensus_queue.cc:1035] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.327924 26311 raft_consensus.cc:2932] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.328377 25134 raft_consensus.cc:2932] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.329157 22845 raft_consensus.cc:2932] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.331267 22444 catalog_manager.cc:5484] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.332163 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 905422dc16e74174b4f88aa87cbee3d3 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:44.332891 26311 raft_consensus.cc:1050] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85: attempting to promote NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e to VOTER
I20240502 07:58:44.333281 26311 consensus_queue.cc:237] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:44.334398 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet 905422dc16e74174b4f88aa87cbee3d3 with delete_type TABLET_DATA_TOMBSTONED (TS 725969d490a04fdfb3ba890c3f5096e3 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:44.334596 25465 tablet_replica.cc:330] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:44.334611 22979 raft_consensus.cc:1261] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.334695 25465 raft_consensus.cc:2229] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.334791 25465 raft_consensus.cc:2258] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.335004 26311 consensus_queue.cc:1035] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.335510 25004 raft_consensus.cc:1261] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.336333 25465 ts_tablet_manager.cc:1875] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:44.337116 26311 consensus_queue.cc:1035] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.337850 22715 raft_consensus.cc:1261] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.338305 25465 ts_tablet_manager.cc:1888] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:44.338383 25465 log.cc:1196] T 905422dc16e74174b4f88aa87cbee3d3 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/905422dc16e74174b4f88aa87cbee3d3
I20240502 07:58:44.338894 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 905422dc16e74174b4f88aa87cbee3d3 (table rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]) successfully deleted
I20240502 07:58:44.339111 26311 consensus_queue.cc:1035] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.340559 25987 raft_consensus.cc:2932] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.340945 25004 raft_consensus.cc:2932] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.342190 22979 raft_consensus.cc:2932] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.342972 22715 raft_consensus.cc:2932] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.343392 22442 catalog_manager.cc:5484] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:44.348378 22584 consensus_queue.cc:237] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:44.349722 25004 raft_consensus.cc:1261] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.349721 22715 raft_consensus.cc:1261] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.350129 26311 consensus_queue.cc:1035] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.350281 26134 consensus_queue.cc:1035] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.352021 24714 raft_consensus.cc:2932] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.352463 25004 raft_consensus.cc:2932] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.353269 22715 raft_consensus.cc:2932] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.353449 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c8c7521bb8414364b927eed84be37621 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:44.354688 22444 catalog_manager.cc:5484] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:44.357894 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet c8c7521bb8414364b927eed84be37621 with delete_type TABLET_DATA_TOMBSTONED (TS 02a92e1c084b4308b9e855a4cc2c295f not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:44.358109 26109 tablet_replica.cc:330] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:44.358235 26109 raft_consensus.cc:2229] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.358361 26109 raft_consensus.cc:2258] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.359539 26109 ts_tablet_manager.cc:1875] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:44.361419 26109 ts_tablet_manager.cc:1888] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:44.361483 26109 log.cc:1196] T c8c7521bb8414364b927eed84be37621 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/c8c7521bb8414364b927eed84be37621
I20240502 07:58:44.361507 22979 consensus_queue.cc:237] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:44.362463 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet c8c7521bb8414364b927eed84be37621 (table rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]) successfully deleted
I20240502 07:58:44.362643 22584 raft_consensus.cc:1261] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.363035 25495 consensus_queue.cc:1035] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.364570 26128 raft_consensus.cc:2932] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:44.365006 22584 raft_consensus.cc:2932] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:44.367034 22441 catalog_manager.cc:5484] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:44.368921 22715 raft_consensus.cc:1261] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.369354 25495 consensus_queue.cc:1035] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 07:58:44.369498 22912 consensus_peers.cc:485] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): Couldn't send request to peer 6d02017318a2461796d431962fe6a537. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 398405a3694c4d29b0627d2d74ebadc6. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:44.370344 25495 raft_consensus.cc:1050] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 to VOTER
I20240502 07:58:44.370718 25495 consensus_queue.cc:237] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:44.371847 22715 raft_consensus.cc:2932] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:44.372256 25134 raft_consensus.cc:1261] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.372581 26128 consensus_queue.cc:1035] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.373981 22584 raft_consensus.cc:1261] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.376008 25670 consensus_queue.cc:1035] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.377964 26134 raft_consensus.cc:1050] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85: attempting to promote NON_VOTER 6d02017318a2461796d431962fe6a537 to VOTER
I20240502 07:58:44.371858 22717 raft_consensus.cc:1261] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.378355 26128 consensus_queue.cc:1035] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.378351 26134 consensus_queue.cc:237] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:44.379631 22716 raft_consensus.cc:1261] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.379751 24871 raft_consensus.cc:1261] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.379868 26342 ts_tablet_manager.cc:902] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:44.380084 22845 raft_consensus.cc:1261] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.380223 26342 tablet_copy_client.cc:308] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:44.380314 25987 consensus_queue.cc:1035] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.380605 25987 consensus_queue.cc:1035] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.381374 22998 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet 1e370c8fe68b4392ba3720533e538de3 from peer 74f9aa91c2c448f980b512fe2e8d672e ({username='slave'} at 127.16.85.134:46615)
I20240502 07:58:44.381461 22998 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet 1e370c8fe68b4392ba3720533e538de3 from peer 74f9aa91c2c448f980b512fe2e8d672e at {username='slave'} at 127.16.85.134:46615: session id = 74f9aa91c2c448f980b512fe2e8d672e-1e370c8fe68b4392ba3720533e538de3
I20240502 07:58:44.381788 26311 consensus_queue.cc:1035] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.382205 22998 tablet_copy_source_session.cc:205] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:44.382848 25669 raft_consensus.cc:2932] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.383189 25134 raft_consensus.cc:2932] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.384987 22715 raft_consensus.cc:2932] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.385327 22440 catalog_manager.cc:5484] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.388316 26342 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1e370c8fe68b4392ba3720533e538de3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.390267 26342 tablet_copy_client.cc:783] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:44.390444 26342 tablet_copy_client.cc:647] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:44.391718 26342 tablet_copy_client.cc:523] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:44.392076 26311 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:44.392774 26342 tablet_bootstrap.cc:492] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:44.392544 24871 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:44.394085 22716 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:44.394702 22845 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:44.394996 22441 catalog_manager.cc:5484] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537 reported cstate change: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:44.399155 22586 raft_consensus.cc:2932] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.400084 22584 consensus_queue.cc:237] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:44.401011 22716 raft_consensus.cc:1261] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.403016 25987 consensus_queue.cc:1035] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.403947 22979 consensus_queue.cc:237] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:44.404636 25987 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:44.405157 25134 raft_consensus.cc:1261] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.405623 25670 consensus_queue.cc:1035] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.406257 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9e5fe86b44744df9b2e4d430fdec716f with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:44.406329 22584 raft_consensus.cc:1261] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.406616 22716 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:44.407246 25670 consensus_queue.cc:1035] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.408053 24871 raft_consensus.cc:1261] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.408092 22442 catalog_manager.cc:5484] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:44.408828 24714 consensus_queue.cc:1035] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.408818 25670 raft_consensus.cc:2932] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.409432 25134 raft_consensus.cc:2932] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.410238 22429 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 73ad3c23d96b43b1894f767ad5db1e2f with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:44.410122 24871 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:44.411854 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:44.412216 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet 9e5fe86b44744df9b2e4d430fdec716f with delete_type TABLET_DATA_TOMBSTONED (TS 7db4178eed174225bfb24f5f9445ce88 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:44.412416 25467 tablet_replica.cc:330] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:44.412541 25467 raft_consensus.cc:2229] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.412678 25467 raft_consensus.cc:2258] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.413542 22441 catalog_manager.cc:5484] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.413861 22586 raft_consensus.cc:2932] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.415596 26284 raft_consensus.cc:1050] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88: attempting to promote NON_VOTER 7eced220cf1642f3873fc385b6191e69 to VOTER
I20240502 07:58:44.415930 26284 consensus_queue.cc:237] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:44.417699 22978 raft_consensus.cc:1261] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.417912 25265 raft_consensus.cc:1261] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.418008 22586 raft_consensus.cc:1261] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.418336 26036 consensus_queue.cc:1035] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.418524 25532 consensus_queue.cc:1035] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.418865 25532 consensus_queue.cc:1035] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.419495 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet 73ad3c23d96b43b1894f767ad5db1e2f with delete_type TABLET_DATA_TOMBSTONED (TS 725969d490a04fdfb3ba890c3f5096e3 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:44.419659 25465 tablet_replica.cc:330] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:44.419773 25465 raft_consensus.cc:2229] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.419937 25465 raft_consensus.cc:2258] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.422628 25467 ts_tablet_manager.cc:1875] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:44.422959 26314 tablet.cc:1722] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:44.423214 26314 tablet.cc:1620] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:44.424002 25465 ts_tablet_manager.cc:1875] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:44.424706 25467 ts_tablet_manager.cc:1888] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:44.424785 25467 log.cc:1196] T 9e5fe86b44744df9b2e4d430fdec716f P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/9e5fe86b44744df9b2e4d430fdec716f
I20240502 07:58:44.425171 26317 tablet.cc:1722] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:44.425462 26317 tablet.cc:1620] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:58:44.425598 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 9e5fe86b44744df9b2e4d430fdec716f (table rebalance_test_table_15 [id=4b1b337aafc04dc69a2e37f348a5652a]) successfully deleted
I20240502 07:58:44.426179 26315 tablet.cc:1722] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:44.426434 26315 tablet.cc:1620] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:44.430392 25465 ts_tablet_manager.cc:1888] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:44.430467 25465 log.cc:1196] T 73ad3c23d96b43b1894f767ad5db1e2f P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/73ad3c23d96b43b1894f767ad5db1e2f
I20240502 07:58:44.431234 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 73ad3c23d96b43b1894f767ad5db1e2f (table rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]) successfully deleted
I20240502 07:58:44.431221 26079 raft_consensus.cc:2932] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.431596 22978 raft_consensus.cc:2932] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.431725 22441 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_8" } modify_external_catalogs: true
I20240502 07:58:44.431818 22441 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_8" } modify_external_catalogs: true
I20240502 07:58:44.433593 25265 raft_consensus.cc:2932] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.434623 22441 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 5124636392c74f56ad54f1ab91b9f7b0
I20240502 07:58:44.434989 22442 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=c403b21f2dee470394eb271267f2a442] alter complete (version 2)
I20240502 07:58:44.435050 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 5124636392c74f56ad54f1ab91b9f7b0 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:44.435227 25468 tablet_replica.cc:330] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:44.435376 25468 raft_consensus.cc:2229] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.435494 25468 raft_consensus.cc:2258] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.435622 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet 5124636392c74f56ad54f1ab91b9f7b0 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:44.435766 25467 tablet_replica.cc:330] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:44.435880 25467 raft_consensus.cc:2229] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:44.435953 25468 ts_tablet_manager.cc:1875] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.436028 25467 raft_consensus.cc:2258] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.436470 25467 ts_tablet_manager.cc:1875] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.438019 25468 ts_tablet_manager.cc:1888] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:44.438078 25468 log.cc:1196] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/5124636392c74f56ad54f1ab91b9f7b0
I20240502 07:58:44.438357 25468 ts_tablet_manager.cc:1909] T 5124636392c74f56ad54f1ab91b9f7b0 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:44.439327 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 5124636392c74f56ad54f1ab91b9f7b0 (table rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]) successfully deleted
I20240502 07:58:44.439719 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet 5124636392c74f56ad54f1ab91b9f7b0 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:44.439882 25467 ts_tablet_manager.cc:1888] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:44.439937 25467 log.cc:1196] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/5124636392c74f56ad54f1ab91b9f7b0
I20240502 07:58:44.439987 25465 tablet_replica.cc:330] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:44.440147 25465 raft_consensus.cc:2229] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.440191 25467 ts_tablet_manager.cc:1909] T 5124636392c74f56ad54f1ab91b9f7b0 P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:44.440270 25465 raft_consensus.cc:2258] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.440728 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 5124636392c74f56ad54f1ab91b9f7b0 (table rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]) successfully deleted
I20240502 07:58:44.440754 25465 ts_tablet_manager.cc:1875] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.440845 22441 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 73ad3c23d96b43b1894f767ad5db1e2f
I20240502 07:58:44.441226 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet 73ad3c23d96b43b1894f767ad5db1e2f with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:44.441380 26109 tablet_replica.cc:330] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:44.441511 26109 raft_consensus.cc:2229] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:44.441684 26109 raft_consensus.cc:2258] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.442238 26109 ts_tablet_manager.cc:1875] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.442445 25465 ts_tablet_manager.cc:1888] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:44.442512 25465 log.cc:1196] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/5124636392c74f56ad54f1ab91b9f7b0
I20240502 07:58:44.442744 25465 ts_tablet_manager.cc:1909] T 5124636392c74f56ad54f1ab91b9f7b0 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:44.443177 26342 tablet_bootstrap.cc:492] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:44.443428 26342 tablet_bootstrap.cc:492] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap complete.
I20240502 07:58:44.443583 26342 ts_tablet_manager.cc:1367] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.051s user 0.024s sys 0.000s
I20240502 07:58:44.443620 22440 catalog_manager.cc:5484] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.444005 26342 raft_consensus.cc:348] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:44.444115 26342 raft_consensus.cc:729] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: LEARNER
I20240502 07:58:44.444216 26109 ts_tablet_manager.cc:1888] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:44.444200 26342 consensus_queue.cc:260] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:44.444278 26109 log.cc:1196] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/73ad3c23d96b43b1894f767ad5db1e2f
I20240502 07:58:44.444545 26109 ts_tablet_manager.cc:1909] T 73ad3c23d96b43b1894f767ad5db1e2f P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:44.445449 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 5124636392c74f56ad54f1ab91b9f7b0 (table rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]) successfully deleted
I20240502 07:58:44.447244 26342 ts_tablet_manager.cc:1398] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:44.447180 22586 raft_consensus.cc:2932] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.447878 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 73ad3c23d96b43b1894f767ad5db1e2f (table rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]) successfully deleted
I20240502 07:58:44.448158 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 73ad3c23d96b43b1894f767ad5db1e2f with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:44.448271 22998 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-1e370c8fe68b4392ba3720533e538de3 received from {username='slave'} at 127.16.85.134:46615
I20240502 07:58:44.448340 22998 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-1e370c8fe68b4392ba3720533e538de3 on tablet 1e370c8fe68b4392ba3720533e538de3 with peer 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:44.448702 25114 tablet_service.cc:1514] Processing DeleteTablet for tablet 73ad3c23d96b43b1894f767ad5db1e2f with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:44.448894 22441 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 4 replicas of tablet b22f8e50a7d24dbdab665e47120602ce
I20240502 07:58:44.449256 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet b22f8e50a7d24dbdab665e47120602ce with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:44.449738 25468 tablet_replica.cc:330] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:44.449744 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet b22f8e50a7d24dbdab665e47120602ce with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:44.449852 26109 tablet_replica.cc:330] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:44.449856 25468 raft_consensus.cc:2229] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.449936 26109 raft_consensus.cc:2229] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:44.449982 25468 raft_consensus.cc:2258] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.450107 26109 raft_consensus.cc:2258] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.450546 26109 ts_tablet_manager.cc:1875] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.450675 25468 ts_tablet_manager.cc:1875] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.451503 22845 consensus_queue.cc:237] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:44.452742 22978 raft_consensus.cc:1261] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.452931 26109 ts_tablet_manager.cc:1888] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:44.452921 25265 raft_consensus.cc:1261] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.452986 26109 log.cc:1196] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/b22f8e50a7d24dbdab665e47120602ce
I20240502 07:58:44.453092 25532 consensus_queue.cc:1035] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.453236 26109 ts_tablet_manager.cc:1909] T b22f8e50a7d24dbdab665e47120602ce P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:44.453744 25532 consensus_queue.cc:1035] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.453783 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet b22f8e50a7d24dbdab665e47120602ce (table rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]) successfully deleted
I20240502 07:58:44.454401 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet b22f8e50a7d24dbdab665e47120602ce with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:44.454753 25467 tablet_replica.cc:330] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:44.454917 25467 raft_consensus.cc:2229] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.455046 24850 tablet_service.cc:1514] Processing DeleteTablet for tablet b22f8e50a7d24dbdab665e47120602ce with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:44.455077 25467 raft_consensus.cc:2258] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.454985 25532 raft_consensus.cc:2932] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.455334 26370 tablet_replica.cc:330] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:44.455490 26370 raft_consensus.cc:2229] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Raft consensus shutting down.
I20240502 07:58:44.455523 25467 ts_tablet_manager.cc:1875] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.455626 26370 raft_consensus.cc:2258] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Raft consensus is shut down!
I20240502 07:58:44.456137 26370 ts_tablet_manager.cc:1875] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.456643 22978 raft_consensus.cc:2932] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.457974 25467 ts_tablet_manager.cc:1888] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:44.458040 25467 log.cc:1196] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/b22f8e50a7d24dbdab665e47120602ce
I20240502 07:58:44.458276 25467 ts_tablet_manager.cc:1909] T b22f8e50a7d24dbdab665e47120602ce P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:44.459120 26370 ts_tablet_manager.cc:1888] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:44.459242 26370 log.cc:1196] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/b22f8e50a7d24dbdab665e47120602ce
I20240502 07:58:44.459228 22436 catalog_manager.cc:5484] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:44.459606 26370 ts_tablet_manager.cc:1909] T b22f8e50a7d24dbdab665e47120602ce P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:44.460090 26369 tablet_replica.cc:330] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:44.460181 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet b22f8e50a7d24dbdab665e47120602ce (table rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]) successfully deleted
I20240502 07:58:44.460258 26369 raft_consensus.cc:2229] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.460417 26369 raft_consensus.cc:2258] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.460669 25265 raft_consensus.cc:2932] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.461143 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 848fc8384054446aa75a05a52488ea57 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:44.461267 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet b22f8e50a7d24dbdab665e47120602ce (table rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]) successfully deleted
I20240502 07:58:44.462760 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 848fc8384054446aa75a05a52488ea57 with delete_type TABLET_DATA_TOMBSTONED (TS 8bea0fe8d8044a2685f92f7a10c9cf85 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:44.464551 25468 ts_tablet_manager.cc:1888] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:44.464669 25468 log.cc:1196] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/73ad3c23d96b43b1894f767ad5db1e2f
I20240502 07:58:44.464987 26311 raft_consensus.cc:1050] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85: attempting to promote NON_VOTER 6d02017318a2461796d431962fe6a537 to VOTER
I20240502 07:58:44.465137 25468 ts_tablet_manager.cc:1909] T 73ad3c23d96b43b1894f767ad5db1e2f P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:44.465325 26311 consensus_queue.cc:237] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:44.465662 26369 ts_tablet_manager.cc:1875] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.465795 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 73ad3c23d96b43b1894f767ad5db1e2f (table rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]) successfully deleted
I20240502 07:58:44.465960 25468 tablet_replica.cc:330] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:44.466068 25468 raft_consensus.cc:2229] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.466171 25468 raft_consensus.cc:2258] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.466747 25468 ts_tablet_manager.cc:1875] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.466848 22978 raft_consensus.cc:1261] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.467020 24871 raft_consensus.cc:1261] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.467186 25987 consensus_queue.cc:1035] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.467311 22716 raft_consensus.cc:1261] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.467885 26134 consensus_queue.cc:1035] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.468204 26134 consensus_queue.cc:1035] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.468474 25468 ts_tablet_manager.cc:1888] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:44.468835 25468 log.cc:1196] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/b22f8e50a7d24dbdab665e47120602ce
I20240502 07:58:44.469130 25468 ts_tablet_manager.cc:1909] T b22f8e50a7d24dbdab665e47120602ce P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:44.469635 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet b22f8e50a7d24dbdab665e47120602ce (table rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]) successfully deleted
I20240502 07:58:44.469796 25468 tablet_replica.cc:330] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:44.469923 25468 raft_consensus.cc:2229] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.470201 25468 raft_consensus.cc:2258] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.471782 26369 ts_tablet_manager.cc:1888] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:44.471851 26369 log.cc:1196] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/73ad3c23d96b43b1894f767ad5db1e2f
I20240502 07:58:44.471855 25468 ts_tablet_manager.cc:1875] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:44.472131 26369 ts_tablet_manager.cc:1909] T 73ad3c23d96b43b1894f767ad5db1e2f P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:44.472697 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet 73ad3c23d96b43b1894f767ad5db1e2f (table rebalance_test_table_8 [id=3af304f23802458e93ee896b8dee72fc]) successfully deleted
I20240502 07:58:44.473532 25987 raft_consensus.cc:2932] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:44.474050 22978 raft_consensus.cc:2932] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:44.474504 25468 ts_tablet_manager.cc:1888] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:44.474556 25468 log.cc:1196] T 848fc8384054446aa75a05a52488ea57 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/848fc8384054446aa75a05a52488ea57
I20240502 07:58:44.476089 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 848fc8384054446aa75a05a52488ea57 (table rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]) successfully deleted
I20240502 07:58:44.476168 24871 raft_consensus.cc:2932] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:44.476465 22716 raft_consensus.cc:2932] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:44.476585 22436 catalog_manager.cc:5484] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:44.480340 22586 consensus_queue.cc:237] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:44.481537 22716 raft_consensus.cc:1261] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.481580 24871 raft_consensus.cc:1261] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.481920 24714 consensus_queue.cc:1035] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.482174 24714 consensus_queue.cc:1035] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.483654 25987 raft_consensus.cc:2932] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:44.484141 22716 raft_consensus.cc:2932] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:44.485183 24871 raft_consensus.cc:2932] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:44.485954 22444 catalog_manager.cc:5484] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:44.487612 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f1830cb65a294734ad80f274b89ac185 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:44.488294 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet f1830cb65a294734ad80f274b89ac185 with delete_type TABLET_DATA_TOMBSTONED (TS 02a92e1c084b4308b9e855a4cc2c295f not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:44.488474 26109 tablet_replica.cc:330] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:44.488611 26109 raft_consensus.cc:2229] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.488803 26109 raft_consensus.cc:2258] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.489471 26109 ts_tablet_manager.cc:1875] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:44.490957 26109 ts_tablet_manager.cc:1888] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:44.491026 26109 log.cc:1196] T f1830cb65a294734ad80f274b89ac185 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/f1830cb65a294734ad80f274b89ac185
I20240502 07:58:44.491433 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet f1830cb65a294734ad80f274b89ac185 (table rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]) successfully deleted
I20240502 07:58:44.494552 25670 raft_consensus.cc:1050] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 7eced220cf1642f3873fc385b6191e69 to VOTER
I20240502 07:58:44.494916 25670 consensus_queue.cc:237] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:44.496160 25265 raft_consensus.cc:1261] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.496218 22845 raft_consensus.cc:1261] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.496474 22716 raft_consensus.cc:1261] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.496595 25495 consensus_queue.cc:1035] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.496815 26128 consensus_queue.cc:1035] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.496958 25669 consensus_queue.cc:1035] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.498715 25669 raft_consensus.cc:2932] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.499146 25265 raft_consensus.cc:2932] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.499231 22845 raft_consensus.cc:2932] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.501331 22715 raft_consensus.cc:2932] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.501960 22436 catalog_manager.cc:5484] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.503105 25557 raft_consensus.cc:1050] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3: attempting to promote NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e to VOTER
I20240502 07:58:44.503569 25557 consensus_queue.cc:237] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:44.505242 22978 consensus_queue.cc:237] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:44.505353 26392 ts_tablet_manager.cc:902] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:44.505841 26392 tablet_copy_client.cc:308] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:44.505972 22586 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.506446 25265 raft_consensus.cc:1261] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.506628 22845 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.506758 22998 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet 9d2283e5d28347519b38e876d840608e from peer 6d02017318a2461796d431962fe6a537 ({username='slave'} at 127.16.85.133:38655)
I20240502 07:58:44.506781 26128 consensus_queue.cc:1035] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.506870 22998 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet 9d2283e5d28347519b38e876d840608e from peer 6d02017318a2461796d431962fe6a537 at {username='slave'} at 127.16.85.133:38655: session id = 6d02017318a2461796d431962fe6a537-9d2283e5d28347519b38e876d840608e
I20240502 07:58:44.507295 25004 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.507284 26150 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.507570 26150 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.507731 26391 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.507808 22998 tablet_copy_source_session.cc:205] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:44.507939 22715 raft_consensus.cc:1261] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.508606 26128 consensus_queue.cc:1035] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.509356 25350 consensus_queue.cc:1035] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:44.510182 25495 raft_consensus.cc:2932] T 88c1176b415c412fb2c824f277e4d3c8 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.510825 26392 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9d2283e5d28347519b38e876d840608e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.510756 25265 raft_consensus.cc:2932] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.512338 22429 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 88c1176b415c412fb2c824f277e4d3c8 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:44.512707 22715 raft_consensus.cc:2932] T 88c1176b415c412fb2c824f277e4d3c8 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.513154 22444 catalog_manager.cc:5484] T 88c1176b415c412fb2c824f277e4d3c8 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.515115 26392 tablet_copy_client.cc:783] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:44.515316 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet 88c1176b415c412fb2c824f277e4d3c8 with delete_type TABLET_DATA_TOMBSTONED (TS 7db4178eed174225bfb24f5f9445ce88 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:44.515617 25467 tablet_replica.cc:330] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:44.515745 25467 raft_consensus.cc:2229] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.515870 25467 raft_consensus.cc:2258] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.516418 25467 ts_tablet_manager.cc:1875] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:44.516458 26391 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.516889 22845 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.518292 22586 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.518486 25467 ts_tablet_manager.cc:1888] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:44.518556 25467 log.cc:1196] T 88c1176b415c412fb2c824f277e4d3c8 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/88c1176b415c412fb2c824f277e4d3c8
I20240502 07:58:44.519098 26392 tablet_copy_client.cc:647] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:44.519372 22441 catalog_manager.cc:5484] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.519598 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 88c1176b415c412fb2c824f277e4d3c8 (table rebalance_test_table_5 [id=cd54bf3678754d7291af91c25c402e8d]) successfully deleted
I20240502 07:58:44.520308 26392 tablet_copy_client.cc:523] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:44.521914 26392 tablet_bootstrap.cc:492] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:44.522415 25004 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.522954 26262 consensus_queue.cc:1035] T 855e40f5f05948eb8902d0a6771fc0bf P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:44.529783 22715 consensus_queue.cc:237] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:44.530787 25004 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.531023 22586 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.531466 25557 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.531606 26391 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.531780 26413 ts_tablet_manager.cc:902] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69: Initiating tablet copy from peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:44.532265 26413 tablet_copy_client.cc:308] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.131:44055
I20240502 07:58:44.533455 26391 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.534811 22428 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e25ba5cc77334a3595a4575fa98476f0 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:44.535040 22586 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.537091 22868 tablet_copy_service.cc:140] P 7db4178eed174225bfb24f5f9445ce88: Received BeginTabletCopySession request for tablet 45499d82e94c47b8b398c159df9fd49b from peer 7eced220cf1642f3873fc385b6191e69 ({username='slave'} at 127.16.85.136:46167)
I20240502 07:58:44.537194 22868 tablet_copy_service.cc:161] P 7db4178eed174225bfb24f5f9445ce88: Beginning new tablet copy session on tablet 45499d82e94c47b8b398c159df9fd49b from peer 7eced220cf1642f3873fc385b6191e69 at {username='slave'} at 127.16.85.136:46167: session id = 7eced220cf1642f3873fc385b6191e69-45499d82e94c47b8b398c159df9fd49b
I20240502 07:58:44.537467 22444 catalog_manager.cc:5484] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.538007 22868 tablet_copy_source_session.cc:205] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:44.538920 25004 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.539634 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet e25ba5cc77334a3595a4575fa98476f0 with delete_type TABLET_DATA_TOMBSTONED (TS 7db4178eed174225bfb24f5f9445ce88 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:44.540139 25467 tablet_replica.cc:330] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:44.540242 25467 raft_consensus.cc:2229] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.540359 25467 raft_consensus.cc:2258] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.540735 26413 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 45499d82e94c47b8b398c159df9fd49b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.541147 25467 ts_tablet_manager.cc:1875] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:44.541374 22441 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:44.542862 26413 tablet_copy_client.cc:783] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:44.542925 25467 ts_tablet_manager.cc:1888] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:44.542995 25467 log.cc:1196] T e25ba5cc77334a3595a4575fa98476f0 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/e25ba5cc77334a3595a4575fa98476f0
I20240502 07:58:44.543017 26413 tablet_copy_client.cc:647] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:44.544982 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet e25ba5cc77334a3595a4575fa98476f0 (table rebalance_test_table_11 [id=0c1156cac15d48028ebf5c07c67ea832]) successfully deleted
I20240502 07:58:44.545536 26413 tablet_copy_client.cc:523] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:44.546519 26413 tablet_bootstrap.cc:492] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:44.548048 26314 tablet.cc:1722] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:44.550063 26315 tablet.cc:1722] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:44.551100 26317 tablet.cc:1722] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:44.551640 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=c403b21f2dee470394eb271267f2a442] alter complete (version 3)
I20240502 07:58:44.558063 26392 tablet_bootstrap.cc:492] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:44.558284 26392 tablet_bootstrap.cc:492] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537: Bootstrap complete.
I20240502 07:58:44.558419 26392 ts_tablet_manager.cc:1367] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.037s user 0.022s sys 0.000s
I20240502 07:58:44.558748 26392 raft_consensus.cc:348] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:44.558846 26392 raft_consensus.cc:729] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: LEARNER
I20240502 07:58:44.558928 26392 consensus_queue.cc:260] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:44.559154 26392 ts_tablet_manager.cc:1398] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:44.559860 22998 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 6d02017318a2461796d431962fe6a537-9d2283e5d28347519b38e876d840608e received from {username='slave'} at 127.16.85.133:38655
I20240502 07:58:44.559928 22998 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 6d02017318a2461796d431962fe6a537-9d2283e5d28347519b38e876d840608e on tablet 9d2283e5d28347519b38e876d840608e with peer 6d02017318a2461796d431962fe6a537
I20240502 07:58:44.572835 25669 raft_consensus.cc:1050] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 to VOTER
I20240502 07:58:44.573259 25669 consensus_queue.cc:237] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:44.574440 25134 raft_consensus.cc:1261] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.574532 22586 raft_consensus.cc:1261] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.574731 22715 raft_consensus.cc:1261] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.574903 25669 consensus_queue.cc:1035] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.575029 25670 consensus_queue.cc:1035] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.575729 25669 consensus_queue.cc:1035] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.579391 25670 raft_consensus.cc:2932] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.581169 24714 raft_consensus.cc:1050] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85: attempting to promote NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 to VOTER
I20240502 07:58:44.581516 24714 consensus_queue.cc:237] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:44.582058 22444 catalog_manager.cc:5484] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:44.582954 22845 raft_consensus.cc:1261] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.583240 25135 raft_consensus.cc:1261] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.583216 25134 raft_consensus.cc:2932] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.584887 22715 raft_consensus.cc:1261] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.585108 25987 consensus_queue.cc:1035] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.585237 26311 consensus_queue.cc:1035] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.585353 26134 consensus_queue.cc:1035] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.587399 22586 raft_consensus.cc:2932] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.587473 22716 raft_consensus.cc:2932] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.588296 26391 consensus_queue.cc:1035] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:44.590587 22978 consensus_queue.cc:237] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:44.591634 25134 raft_consensus.cc:1261] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.591926 25669 consensus_queue.cc:1035] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.592020 22586 raft_consensus.cc:1261] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.592528 26128 consensus_queue.cc:1035] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.593444 25669 raft_consensus.cc:2932] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.595228 22429 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1d382df270004c128dc5718be32532ba with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:44.595214 25134 raft_consensus.cc:2932] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.595582 22441 catalog_manager.cc:5484] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:44.596809 22586 raft_consensus.cc:2932] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.597579 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet 1d382df270004c128dc5718be32532ba with delete_type TABLET_DATA_TOMBSTONED (TS 725969d490a04fdfb3ba890c3f5096e3 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:44.597805 25465 tablet_replica.cc:330] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:44.597921 25465 raft_consensus.cc:2229] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.598071 25465 raft_consensus.cc:2258] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.598496 26311 consensus_queue.cc:1035] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:44.598802 25465 ts_tablet_manager.cc:1875] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:44.600616 25465 ts_tablet_manager.cc:1888] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:44.600687 25465 log.cc:1196] T 1d382df270004c128dc5718be32532ba P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/1d382df270004c128dc5718be32532ba
I20240502 07:58:44.601168 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 1d382df270004c128dc5718be32532ba (table rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]) successfully deleted
I20240502 07:58:44.601229 24871 raft_consensus.cc:3037] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.602793 26150 consensus_queue.cc:1035] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:44.611052 26134 consensus_queue.cc:1035] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:44.611819 26134 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.612957 22715 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.614387 22845 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.614627 25134 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.615607 22444 catalog_manager.cc:5484] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.617493 25669 raft_consensus.cc:1050] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 7eced220cf1642f3873fc385b6191e69 to VOTER
I20240502 07:58:44.617796 25669 consensus_queue.cc:237] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:44.618889 22586 consensus_queue.cc:237] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:44.618932 25265 raft_consensus.cc:1261] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.619225 25495 consensus_queue.cc:1035] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.619307 22845 raft_consensus.cc:1261] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.619663 25495 consensus_queue.cc:1035] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.619829 22715 raft_consensus.cc:1261] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.620206 25495 consensus_queue.cc:1035] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.620420 25134 raft_consensus.cc:1261] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.620534 22847 raft_consensus.cc:1261] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.621332 26311 consensus_queue.cc:1035] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.621610 26311 consensus_queue.cc:1035] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.622969 25987 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.623523 25134 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.624114 25670 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.624426 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet dd815609d7324cc588f02b6cb40cfbfd with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:44.624522 22715 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.625622 22441 catalog_manager.cc:5484] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.626504 22442 catalog_manager.cc:5484] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:44.626893 22845 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.627825 22847 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.628459 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet dd815609d7324cc588f02b6cb40cfbfd with delete_type TABLET_DATA_TOMBSTONED (TS 725969d490a04fdfb3ba890c3f5096e3 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:44.628684 25465 tablet_replica.cc:330] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:44.628816 25465 raft_consensus.cc:2229] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.628958 25465 raft_consensus.cc:2258] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.630789 25465 ts_tablet_manager.cc:1875] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:44.633702 25465 ts_tablet_manager.cc:1888] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:44.633771 25465 log.cc:1196] T dd815609d7324cc588f02b6cb40cfbfd P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/dd815609d7324cc588f02b6cb40cfbfd
I20240502 07:58:44.634256 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet dd815609d7324cc588f02b6cb40cfbfd (table rebalance_test_table_17 [id=35a66cc6bd364d818b35bceada233a98]) successfully deleted
I20240502 07:58:44.634835 25265 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.636544 22978 consensus_queue.cc:237] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:44.637564 25265 raft_consensus.cc:1261] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.637647 22715 raft_consensus.cc:1261] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.638020 25495 consensus_queue.cc:1035] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.638278 25495 consensus_queue.cc:1035] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.639931 25670 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.640550 22715 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.640919 26413 tablet_bootstrap.cc:492] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:44.641597 22429 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cfff02438656430686be0f03f5c0b8ab with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:44.641127 26413 tablet_bootstrap.cc:492] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69: Bootstrap complete.
I20240502 07:58:44.641866 26413 ts_tablet_manager.cc:1367] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.095s user 0.022s sys 0.001s
I20240502 07:58:44.642386 25265 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.642658 22442 catalog_manager.cc:5484] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:44.643141 26413 raft_consensus.cc:348] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:44.643249 26413 raft_consensus.cc:729] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: LEARNER
I20240502 07:58:44.643339 26413 consensus_queue.cc:260] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:44.643615 26413 ts_tablet_manager.cc:1398] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 07:58:44.644055 22868 tablet_copy_service.cc:342] P 7db4178eed174225bfb24f5f9445ce88: Request end of tablet copy session 7eced220cf1642f3873fc385b6191e69-45499d82e94c47b8b398c159df9fd49b received from {username='slave'} at 127.16.85.136:46167
I20240502 07:58:44.644136 22868 tablet_copy_service.cc:433] P 7db4178eed174225bfb24f5f9445ce88: ending tablet copy session 7eced220cf1642f3873fc385b6191e69-45499d82e94c47b8b398c159df9fd49b on tablet 45499d82e94c47b8b398c159df9fd49b with peer 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:44.644809 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet cfff02438656430686be0f03f5c0b8ab with delete_type TABLET_DATA_TOMBSTONED (TS 7db4178eed174225bfb24f5f9445ce88 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:44.644979 25467 tablet_replica.cc:330] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:44.645093 25467 raft_consensus.cc:2229] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.645220 25467 raft_consensus.cc:2258] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.645859 25467 ts_tablet_manager.cc:1875] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:44.647580 25467 ts_tablet_manager.cc:1888] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:44.647634 25467 log.cc:1196] T cfff02438656430686be0f03f5c0b8ab P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/cfff02438656430686be0f03f5c0b8ab
I20240502 07:58:44.648272 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet cfff02438656430686be0f03f5c0b8ab (table rebalance_test_table_9 [id=bc838e66a1294569a905266b5619a7d7]) successfully deleted
I20240502 07:58:44.654479 22441 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:44.654587 22441 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:44.656416 22441 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet efadbab41bd24244bcc0247234ea27a3
I20240502 07:58:44.657076 25114 tablet_service.cc:1514] Processing DeleteTablet for tablet efadbab41bd24244bcc0247234ea27a3 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:44.657235 26369 tablet_replica.cc:330] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:44.657241 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet efadbab41bd24244bcc0247234ea27a3 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:44.657354 26369 raft_consensus.cc:2229] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.657413 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet efadbab41bd24244bcc0247234ea27a3 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:44.657460 26369 raft_consensus.cc:2258] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.657454 26473 tablet_replica.cc:330] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:44.657550 25467 tablet_replica.cc:330] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:44.657650 25467 raft_consensus.cc:2229] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.657750 25467 raft_consensus.cc:2258] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.658214 26473 raft_consensus.cc:2229] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:44.658269 25467 ts_tablet_manager.cc:1875] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.658392 26369 ts_tablet_manager.cc:1875] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.658409 26473 raft_consensus.cc:2258] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.659219 26473 ts_tablet_manager.cc:1875] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.660401 25467 ts_tablet_manager.cc:1888] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:44.660467 25467 log.cc:1196] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/efadbab41bd24244bcc0247234ea27a3
I20240502 07:58:44.660715 25467 ts_tablet_manager.cc:1909] T efadbab41bd24244bcc0247234ea27a3 P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:44.661538 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet efadbab41bd24244bcc0247234ea27a3 (table rebalancer_dynamic_table_new_name [id=c403b21f2dee470394eb271267f2a442]) successfully deleted
I20240502 07:58:44.661633 26369 ts_tablet_manager.cc:1888] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:44.661693 26369 log.cc:1196] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/efadbab41bd24244bcc0247234ea27a3
I20240502 07:58:44.661737 26473 ts_tablet_manager.cc:1888] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:44.661787 26473 log.cc:1196] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/efadbab41bd24244bcc0247234ea27a3
I20240502 07:58:44.661942 26369 ts_tablet_manager.cc:1909] T efadbab41bd24244bcc0247234ea27a3 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:44.661995 26473 ts_tablet_manager.cc:1909] T efadbab41bd24244bcc0247234ea27a3 P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:58:44.662487 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet efadbab41bd24244bcc0247234ea27a3 (table rebalancer_dynamic_table_new_name [id=c403b21f2dee470394eb271267f2a442]) successfully deleted
I20240502 07:58:44.662634 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet efadbab41bd24244bcc0247234ea27a3 (table rebalancer_dynamic_table_new_name [id=c403b21f2dee470394eb271267f2a442]) successfully deleted
I20240502 07:58:44.679469 22586 consensus_queue.cc:237] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:44.680542 22845 raft_consensus.cc:1261] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.680833 22715 raft_consensus.cc:1261] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.680863 26311 consensus_queue.cc:1035] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.681161 26311 consensus_queue.cc:1035] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 07:58:44.681293 22521 consensus_peers.cc:485] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Couldn't send request to peer 7eced220cf1642f3873fc385b6191e69. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c53e8b3bd774466dbafe4994472975bc. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:44.682518 26311 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:44.683298 22715 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:44.683988 22845 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:44.685228 22441 catalog_manager.cc:5484] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:44.692137 22586 consensus_queue.cc:237] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:44.692615 25669 raft_consensus.cc:1050] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e to VOTER
I20240502 07:58:44.693006 25669 consensus_queue.cc:237] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:44.693239 22978 raft_consensus.cc:1261] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.693464 22715 raft_consensus.cc:1261] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.693615 26311 consensus_queue.cc:1035] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 07:58:44.693746 22521 consensus_peers.cc:485] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b583c8775a874ad1a89c82c00c05bbbd. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:44.693905 26311 consensus_queue.cc:1035] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.694212 22715 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.694437 25004 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.694458 22845 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.694772 25670 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.694940 26128 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.695160 25669 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.695773 26311 raft_consensus.cc:2932] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:44.696776 22715 raft_consensus.cc:2932] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:44.697269 22978 raft_consensus.cc:2932] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:44.697995 25669 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.698246 22715 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.698941 22442 catalog_manager.cc:5484] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:44.701021 22845 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.701964 22444 catalog_manager.cc:5484] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.702863 25004 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.706228 22978 consensus_queue.cc:237] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:44.707198 22845 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.707554 25495 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.707695 25004 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.708052 25495 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.708909 25670 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.710073 22429 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f1fa312fe8e44d23970c5e3ba98c2c58 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:44.712965 22847 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.713374 22442 catalog_manager.cc:5484] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:44.715040 25004 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:44.716275 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet f1fa312fe8e44d23970c5e3ba98c2c58 with delete_type TABLET_DATA_TOMBSTONED (TS 725969d490a04fdfb3ba890c3f5096e3 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:44.716425 25465 tablet_replica.cc:330] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:44.716526 25465 raft_consensus.cc:2229] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.716635 25465 raft_consensus.cc:2258] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.717219 25465 ts_tablet_manager.cc:1875] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:44.718844 25465 ts_tablet_manager.cc:1888] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:44.718902 25465 log.cc:1196] T f1fa312fe8e44d23970c5e3ba98c2c58 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/f1fa312fe8e44d23970c5e3ba98c2c58
I20240502 07:58:44.719303 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet f1fa312fe8e44d23970c5e3ba98c2c58 (table rebalance_test_table_13 [id=2882412f9f504bfc93c45c294c9ace87]) successfully deleted
I20240502 07:58:44.721817 22847 consensus_queue.cc:237] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:44.723007 22715 raft_consensus.cc:1261] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.723412 26079 consensus_queue.cc:1035] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.723496 22586 raft_consensus.cc:1261] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.723829 26079 consensus_queue.cc:1035] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.725201 26036 raft_consensus.cc:2932] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:44.725492 22716 raft_consensus.cc:2932] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:44.727093 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_6"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:44.727995 22436 catalog_manager.cc:5484] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
W20240502 07:58:44.735450 22784 consensus_peers.cc:485] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): Couldn't send request to peer 4654231dfe994c1a9d2e37d9c3d75699. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 015ed33d8c634473997eaa60e91269a2. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:44.736380 22584 consensus_queue.cc:237] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
W20240502 07:58:44.737560 22519 consensus_peers.cc:485] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): Couldn't send request to peer 6d02017318a2461796d431962fe6a537. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 887c4c76980d4672aec02b2dbd4d8c89. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:44.737838 22978 raft_consensus.cc:1261] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.738104 25987 consensus_queue.cc:1035] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.738510 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 07409f4ddd0040fc98e6b2064e2df523 (DEFAULT_TABLE table=rebalancer_extra_table_6 [id=4aefdd54f1e145688b445ce3d02fb21e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.738636 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 07409f4ddd0040fc98e6b2064e2df523 (DEFAULT_TABLE table=rebalancer_extra_table_6 [id=4aefdd54f1e145688b445ce3d02fb21e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.738754 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 07409f4ddd0040fc98e6b2064e2df523. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.738849 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 07409f4ddd0040fc98e6b2064e2df523. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.739035 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 66ac69e448574c7ea5101014e0dd425d (DEFAULT_TABLE table=rebalancer_extra_table_6 [id=4aefdd54f1e145688b445ce3d02fb21e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.739207 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 66ac69e448574c7ea5101014e0dd425d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.740648 25114 tablet_service.cc:1467] Processing CreateTablet for tablet 66ac69e448574c7ea5101014e0dd425d (DEFAULT_TABLE table=rebalancer_extra_table_6 [id=4aefdd54f1e145688b445ce3d02fb21e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.740895 25114 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 66ac69e448574c7ea5101014e0dd425d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.741014 26524 tablet_bootstrap.cc:492] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:44.741042 24983 tablet_service.cc:1467] Processing CreateTablet for tablet 4e687b2e36574856a45752104ce83953 (DEFAULT_TABLE table=rebalancer_extra_table_6 [id=4aefdd54f1e145688b445ce3d02fb21e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.741067 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 4e687b2e36574856a45752104ce83953 (DEFAULT_TABLE table=rebalancer_extra_table_6 [id=4aefdd54f1e145688b445ce3d02fb21e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.741257 24983 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4e687b2e36574856a45752104ce83953. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.741254 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4e687b2e36574856a45752104ce83953. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.741849 26524 tablet_bootstrap.cc:654] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.742748 26527 tablet_bootstrap.cc:492] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:44.742898 24984 tablet_service.cc:1467] Processing CreateTablet for tablet 07409f4ddd0040fc98e6b2064e2df523 (DEFAULT_TABLE table=rebalancer_extra_table_6 [id=4aefdd54f1e145688b445ce3d02fb21e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.743114 24984 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 07409f4ddd0040fc98e6b2064e2df523. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.743441 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 4e687b2e36574856a45752104ce83953 (DEFAULT_TABLE table=rebalancer_extra_table_6 [id=4aefdd54f1e145688b445ce3d02fb21e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.743654 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4e687b2e36574856a45752104ce83953. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.743695 26527 tablet_bootstrap.cc:654] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.745028 22847 raft_consensus.cc:1261] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.745023 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 66ac69e448574c7ea5101014e0dd425d (DEFAULT_TABLE table=rebalancer_extra_table_6 [id=4aefdd54f1e145688b445ce3d02fb21e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:44.745224 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 66ac69e448574c7ea5101014e0dd425d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.745615 26134 consensus_queue.cc:1035] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.747282 26392 ts_tablet_manager.cc:902] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:44.747644 26525 tablet_bootstrap.cc:492] T 07409f4ddd0040fc98e6b2064e2df523 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:44.747645 26392 tablet_copy_client.cc:308] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:44.748443 26525 tablet_bootstrap.cc:654] T 07409f4ddd0040fc98e6b2064e2df523 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.748435 22998 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet 398405a3694c4d29b0627d2d74ebadc6 from peer 6d02017318a2461796d431962fe6a537 ({username='slave'} at 127.16.85.133:38655)
I20240502 07:58:44.748508 22998 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet 398405a3694c4d29b0627d2d74ebadc6 from peer 6d02017318a2461796d431962fe6a537 at {username='slave'} at 127.16.85.133:38655: session id = 6d02017318a2461796d431962fe6a537-398405a3694c4d29b0627d2d74ebadc6
I20240502 07:58:44.749302 22998 tablet_copy_source_session.cc:205] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:44.749981 26529 tablet_bootstrap.cc:492] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:44.750125 25987 raft_consensus.cc:2932] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:44.750500 22586 raft_consensus.cc:2932] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:44.752151 26529 tablet_bootstrap.cc:654] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.752285 22978 raft_consensus.cc:2932] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:44.754001 22442 catalog_manager.cc:5484] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:44.754595 26392 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 398405a3694c4d29b0627d2d74ebadc6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.754568 22847 raft_consensus.cc:2932] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:44.756047 26528 tablet_bootstrap.cc:492] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:44.756294 26392 tablet_copy_client.cc:783] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:44.756418 26392 tablet_copy_client.cc:647] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:44.757300 26528 tablet_bootstrap.cc:654] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.757854 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:44.763170 26529 tablet_bootstrap.cc:492] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:44.763340 26529 ts_tablet_manager.cc:1367] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.013s user 0.002s sys 0.000s
I20240502 07:58:44.763911 24984 tablet_service.cc:1467] Processing CreateTablet for tablet 92d6b875b1a942dc94cbf54dd42fd2f6 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=2f175365958540dbb77eef3ec4df49bd]), partition=
I20240502 07:58:44.764094 24984 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 92d6b875b1a942dc94cbf54dd42fd2f6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.764089 25114 tablet_service.cc:1467] Processing CreateTablet for tablet 92d6b875b1a942dc94cbf54dd42fd2f6 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=2f175365958540dbb77eef3ec4df49bd]), partition=
I20240502 07:58:44.764261 25114 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 92d6b875b1a942dc94cbf54dd42fd2f6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.764276 26525 tablet_bootstrap.cc:492] T 07409f4ddd0040fc98e6b2064e2df523 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:44.764333 26525 ts_tablet_manager.cc:1367] T 07409f4ddd0040fc98e6b2064e2df523 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.017s user 0.001s sys 0.000s
I20240502 07:58:44.764650 26525 raft_consensus.cc:348] T 07409f4ddd0040fc98e6b2064e2df523 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:44.764753 26525 raft_consensus.cc:374] T 07409f4ddd0040fc98e6b2064e2df523 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.764777 26525 raft_consensus.cc:729] T 07409f4ddd0040fc98e6b2064e2df523 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.764886 26525 consensus_queue.cc:260] T 07409f4ddd0040fc98e6b2064e2df523 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:44.765511 26525 ts_tablet_manager.cc:1398] T 07409f4ddd0040fc98e6b2064e2df523 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:44.765635 26525 tablet_bootstrap.cc:492] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:44.766517 26525 tablet_bootstrap.cc:654] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.766532 26392 tablet_copy_client.cc:523] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:44.767077 26529 raft_consensus.cc:348] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.767196 26529 raft_consensus.cc:374] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.767248 26529 raft_consensus.cc:729] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.767505 26392 tablet_bootstrap.cc:492] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:44.767499 26529 consensus_queue.cc:260] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.767968 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 92d6b875b1a942dc94cbf54dd42fd2f6 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=2f175365958540dbb77eef3ec4df49bd]), partition=
I20240502 07:58:44.768136 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 92d6b875b1a942dc94cbf54dd42fd2f6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:44.768252 26529 ts_tablet_manager.cc:1398] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:44.768380 26529 tablet_bootstrap.cc:492] T 07409f4ddd0040fc98e6b2064e2df523 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:44.769243 26529 tablet_bootstrap.cc:654] T 07409f4ddd0040fc98e6b2064e2df523 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.770772 26528 tablet_bootstrap.cc:492] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:44.770913 26528 ts_tablet_manager.cc:1367] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.015s user 0.002s sys 0.000s
I20240502 07:58:44.771179 26529 tablet_bootstrap.cc:492] T 07409f4ddd0040fc98e6b2064e2df523 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:44.771394 26529 ts_tablet_manager.cc:1367] T 07409f4ddd0040fc98e6b2064e2df523 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240502 07:58:44.771373 26528 raft_consensus.cc:348] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.771549 26528 raft_consensus.cc:374] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.771611 26528 raft_consensus.cc:729] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.771819 26529 raft_consensus.cc:348] T 07409f4ddd0040fc98e6b2064e2df523 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:44.771937 26529 raft_consensus.cc:374] T 07409f4ddd0040fc98e6b2064e2df523 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.771914 26528 consensus_queue.cc:260] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.771994 26529 raft_consensus.cc:729] T 07409f4ddd0040fc98e6b2064e2df523 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.772173 26529 consensus_queue.cc:260] T 07409f4ddd0040fc98e6b2064e2df523 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:44.772425 26529 ts_tablet_manager.cc:1398] T 07409f4ddd0040fc98e6b2064e2df523 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:44.772809 26529 tablet_bootstrap.cc:492] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:44.773209 26528 ts_tablet_manager.cc:1398] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:44.773406 26528 tablet_bootstrap.cc:492] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:44.773626 26529 tablet_bootstrap.cc:654] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.774911 26528 tablet_bootstrap.cc:654] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.776305 26528 tablet_bootstrap.cc:492] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:44.776394 26528 ts_tablet_manager.cc:1367] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:44.776768 26528 raft_consensus.cc:348] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:44.776870 26528 raft_consensus.cc:374] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.776932 26528 raft_consensus.cc:729] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.777093 26528 consensus_queue.cc:260] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:44.777372 26528 ts_tablet_manager.cc:1398] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:44.777547 26524 tablet_bootstrap.cc:492] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:44.777571 26528 tablet_bootstrap.cc:492] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:44.777631 26524 ts_tablet_manager.cc:1367] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.037s user 0.002s sys 0.000s
I20240502 07:58:44.778070 26524 raft_consensus.cc:348] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:44.778167 26524 raft_consensus.cc:374] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.778203 26524 raft_consensus.cc:729] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.778314 26524 consensus_queue.cc:260] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:44.778550 26528 tablet_bootstrap.cc:654] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.778561 26524 ts_tablet_manager.cc:1398] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:44.779187 26527 tablet_bootstrap.cc:492] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:44.779273 26527 ts_tablet_manager.cc:1367] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.037s user 0.002s sys 0.000s
I20240502 07:58:44.779789 26527 raft_consensus.cc:348] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:44.779942 26527 raft_consensus.cc:374] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.779979 26527 raft_consensus.cc:729] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.780094 26527 consensus_queue.cc:260] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:44.780341 26527 ts_tablet_manager.cc:1398] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:44.781685 26524 tablet_bootstrap.cc:492] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:44.782485 26529 tablet_bootstrap.cc:492] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:44.782797 26529 ts_tablet_manager.cc:1367] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.010s user 0.000s sys 0.002s
I20240502 07:58:44.782824 26524 tablet_bootstrap.cc:654] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.783644 26529 raft_consensus.cc:348] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:44.783789 26529 raft_consensus.cc:374] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.783864 26524 tablet_bootstrap.cc:492] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:44.783874 26529 raft_consensus.cc:729] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.784008 26524 ts_tablet_manager.cc:1367] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:44.784013 26529 consensus_queue.cc:260] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:44.784477 26524 raft_consensus.cc:348] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:44.784629 26524 raft_consensus.cc:374] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.784740 26524 raft_consensus.cc:729] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.784876 26524 consensus_queue.cc:260] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:44.785157 26524 ts_tablet_manager.cc:1398] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:44.785594 26529 ts_tablet_manager.cc:1398] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:44.785972 22978 consensus_queue.cc:237] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:44.786562 26527 tablet_bootstrap.cc:492] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:44.787400 22586 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:44.787497 22914 consensus_peers.cc:485] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Couldn't send request to peer 7eced220cf1642f3873fc385b6191e69. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ca85b40df34d4e1c91a55d7dd69e1494. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:44.787513 26527 tablet_bootstrap.cc:654] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:44.787827 22847 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.787982 25495 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.788455 26527 tablet_bootstrap.cc:492] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:44.788491 25495 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.788550 26527 ts_tablet_manager.cc:1367] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:44.789083 26527 raft_consensus.cc:348] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:44.789186 26527 raft_consensus.cc:374] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.789214 26527 raft_consensus.cc:729] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.789315 26527 consensus_queue.cc:260] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:44.789582 26527 ts_tablet_manager.cc:1398] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:44.790431 26528 tablet_bootstrap.cc:492] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:44.790513 26528 ts_tablet_manager.cc:1367] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.013s user 0.002s sys 0.000s
I20240502 07:58:44.790421 25495 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:44.790935 26528 raft_consensus.cc:348] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:44.791015 26528 raft_consensus.cc:374] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.791042 26528 raft_consensus.cc:729] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.791139 26528 consensus_queue.cc:260] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:44.791385 26528 ts_tablet_manager.cc:1398] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:44.792404 22586 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:44.794029 26525 tablet_bootstrap.cc:492] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:44.794118 26525 ts_tablet_manager.cc:1367] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.029s user 0.001s sys 0.001s
I20240502 07:58:44.794585 26525 raft_consensus.cc:348] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.794557 22847 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:44.794683 26525 raft_consensus.cc:374] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:44.794711 26525 raft_consensus.cc:729] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:44.794829 26525 consensus_queue.cc:260] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.795059 26525 ts_tablet_manager.cc:1398] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:44.797422 22444 catalog_manager.cc:5484] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:44.804760 25557 raft_consensus.cc:1050] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3: attempting to promote NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 to VOTER
I20240502 07:58:44.805081 22715 consensus_queue.cc:237] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:44.805122 25557 consensus_queue.cc:237] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:44.806293 22978 raft_consensus.cc:1261] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.806360 22847 raft_consensus.cc:1261] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:44.806607 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 539cf5d4596f481cb03ab4d13a19198c. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:44.806704 26150 consensus_queue.cc:1035] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.807070 26150 consensus_queue.cc:1035] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.807695 22979 raft_consensus.cc:1261] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.807859 22586 raft_consensus.cc:1261] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.808202 25134 raft_consensus.cc:1261] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:44.808547 26150 consensus_queue.cc:1035] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.808729 26391 consensus_queue.cc:1035] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.808935 26389 consensus_queue.cc:1035] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:44.809532 26389 raft_consensus.cc:2932] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:44.811244 22978 raft_consensus.cc:2932] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:44.813655 22440 catalog_manager.cc:5484] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:44.814402 22847 raft_consensus.cc:2932] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:44.816699 26392 tablet_bootstrap.cc:492] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:44.817085 26392 tablet_bootstrap.cc:492] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537: Bootstrap complete.
I20240502 07:58:44.817358 26392 ts_tablet_manager.cc:1367] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.050s user 0.021s sys 0.004s
I20240502 07:58:44.817819 26392 raft_consensus.cc:348] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:44.818004 26392 raft_consensus.cc:729] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: LEARNER
I20240502 07:58:44.818163 26392 consensus_queue.cc:260] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:44.818336 26391 raft_consensus.cc:2932] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.819041 26392 ts_tablet_manager.cc:1398] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:44.820222 22998 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 6d02017318a2461796d431962fe6a537-398405a3694c4d29b0627d2d74ebadc6 received from {username='slave'} at 127.16.85.133:38655
I20240502 07:58:44.820303 22998 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 6d02017318a2461796d431962fe6a537-398405a3694c4d29b0627d2d74ebadc6 on tablet 398405a3694c4d29b0627d2d74ebadc6 with peer 6d02017318a2461796d431962fe6a537
I20240502 07:58:44.820907 22978 raft_consensus.cc:2932] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.821094 25532 raft_consensus.cc:482] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:44.820984 22444 catalog_manager.cc:5484] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:44.821168 25532 raft_consensus.cc:504] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:44.821504 25532 leader_election.cc:288] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:44.821831 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "66ac69e448574c7ea5101014e0dd425d" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:44.821952 25135 raft_consensus.cc:2454] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:44.821976 25134 raft_consensus.cc:2932] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.822543 22784 leader_election.cc:302] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:44.822678 25532 raft_consensus.cc:2781] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:44.822729 25532 raft_consensus.cc:482] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:44.822775 25532 raft_consensus.cc:3037] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.823657 25532 raft_consensus.cc:504] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:44.823964 25532 leader_election.cc:288] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:44.824414 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "66ac69e448574c7ea5101014e0dd425d" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:44.824509 25134 raft_consensus.cc:3037] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.824755 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "66ac69e448574c7ea5101014e0dd425d" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:44.824873 25265 raft_consensus.cc:2454] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:44.825265 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "66ac69e448574c7ea5101014e0dd425d" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:44.825340 25134 raft_consensus.cc:2454] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:44.825361 25265 raft_consensus.cc:3037] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.825618 22784 leader_election.cc:302] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:44.825786 25532 raft_consensus.cc:2781] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:44.825847 25532 raft_consensus.cc:686] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:44.825984 25532 consensus_queue.cc:237] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:44.826244 25265 raft_consensus.cc:2454] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:44.828308 22586 raft_consensus.cc:2932] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.828500 22715 consensus_queue.cc:237] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:44.829669 22584 raft_consensus.cc:1261] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.829866 25134 raft_consensus.cc:1261] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:44.830163 26150 consensus_queue.cc:1035] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.830482 26150 consensus_queue.cc:1035] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:44.831662 22440 catalog_manager.cc:5484] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:44.834375 25557 raft_consensus.cc:2932] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.834735 22584 raft_consensus.cc:2932] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.834786 25134 raft_consensus.cc:2932] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:44.836234 22428 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0e01743578f947cca1db2baec4fdee12 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:44.836946 22978 consensus_queue.cc:237] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:44.837047 22440 catalog_manager.cc:5484] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:44.838343 22847 raft_consensus.cc:1261] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.838413 22715 raft_consensus.cc:1261] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.838797 25670 consensus_queue.cc:1035] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 07:58:44.838971 22912 consensus_peers.cc:485] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): Couldn't send request to peer 6d02017318a2461796d431962fe6a537. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e5d5f67e026344998d5fb079f532d076. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:44.839147 26128 consensus_queue.cc:1035] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.839557 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet 0e01743578f947cca1db2baec4fdee12 with delete_type TABLET_DATA_TOMBSTONED (TS 02a92e1c084b4308b9e855a4cc2c295f not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:44.839742 26109 tablet_replica.cc:330] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:44.839866 26109 raft_consensus.cc:2229] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.840003 26109 raft_consensus.cc:2258] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.840957 26109 ts_tablet_manager.cc:1875] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:44.841333 26128 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:44.843088 26109 ts_tablet_manager.cc:1888] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:44.843147 26109 log.cc:1196] T 0e01743578f947cca1db2baec4fdee12 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/0e01743578f947cca1db2baec4fdee12
I20240502 07:58:44.843073 22715 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:44.843772 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 0e01743578f947cca1db2baec4fdee12 (table rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]) successfully deleted
I20240502 07:58:44.844440 22436 catalog_manager.cc:5484] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:44.844868 25532 raft_consensus.cc:482] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:44.844974 25532 raft_consensus.cc:504] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:44.845052 22847 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:44.845752 25532 leader_election.cc:288] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:44.846647 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "07409f4ddd0040fc98e6b2064e2df523" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:44.846792 24871 raft_consensus.cc:2454] T 07409f4ddd0040fc98e6b2064e2df523 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:44.847030 22781 leader_election.cc:302] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:44.847178 25532 raft_consensus.cc:2781] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:44.847226 25532 raft_consensus.cc:482] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:44.847266 25532 raft_consensus.cc:3037] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.848065 25532 raft_consensus.cc:504] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:44.848536 25532 leader_election.cc:288] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:44.848706 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "07409f4ddd0040fc98e6b2064e2df523" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:44.848822 24871 raft_consensus.cc:3037] T 07409f4ddd0040fc98e6b2064e2df523 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.849574 24871 raft_consensus.cc:2454] T 07409f4ddd0040fc98e6b2064e2df523 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:44.850277 22781 leader_election.cc:302] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:44.850433 25532 raft_consensus.cc:2781] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:44.850517 25532 raft_consensus.cc:686] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:44.850638 25532 consensus_queue.cc:237] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:44.851137 25004 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "07409f4ddd0040fc98e6b2064e2df523" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e" is_pre_election: true
I20240502 07:58:44.851174 25003 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "07409f4ddd0040fc98e6b2064e2df523" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e"
I20240502 07:58:44.851274 25004 raft_consensus.cc:2454] T 07409f4ddd0040fc98e6b2064e2df523 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:44.852429 22444 catalog_manager.cc:5484] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:44.854722 25004 raft_consensus.cc:1203] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:44.855608 22978 consensus_queue.cc:237] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:44.857056 22715 raft_consensus.cc:1261] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:44.857093 22915 consensus_peers.cc:485] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): Couldn't send request to peer 4654231dfe994c1a9d2e37d9c3d75699. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e24e4e639b0649e3ac135ac9358a0398. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:44.857311 22847 raft_consensus.cc:1261] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.857565 25670 consensus_queue.cc:1035] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.857712 25495 consensus_queue.cc:1035] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.859314 25669 raft_consensus.cc:2932] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:44.859771 22715 raft_consensus.cc:2932] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:44.859797 22847 raft_consensus.cc:2932] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:44.862006 22444 catalog_manager.cc:5484] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:44.868440 22978 consensus_queue.cc:237] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:44.869648 22715 raft_consensus.cc:1261] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.869901 22584 raft_consensus.cc:1261] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.869959 26128 consensus_queue.cc:1035] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 07:58:44.870101 22914 consensus_peers.cc:485] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: eb0a7b1308114769b556795ae18a746e. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:44.870223 25495 consensus_queue.cc:1035] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.871543 26128 raft_consensus.cc:2932] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:44.871933 22715 raft_consensus.cc:2932] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:44.872362 22584 raft_consensus.cc:2932] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:44.874529 22440 catalog_manager.cc:5484] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:44.882237 22847 consensus_queue.cc:237] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:44.883401 22978 raft_consensus.cc:1261] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.883507 22715 raft_consensus.cc:1261] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:44.883529 22783 consensus_peers.cc:485] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 -> Peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Couldn't send request to peer 7eced220cf1642f3873fc385b6191e69. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: abfc26b86b17464fa7cf0b13ab73861c. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:44.883816 26079 consensus_queue.cc:1035] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.884011 26284 consensus_queue.cc:1035] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.884806 25331 raft_consensus.cc:482] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:44.884923 25331 raft_consensus.cc:504] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.885449 25331 leader_election.cc:288] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:44.885778 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4e687b2e36574856a45752104ce83953" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:44.885897 24871 raft_consensus.cc:2454] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:44.885911 25532 raft_consensus.cc:2932] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:44.886240 25199 leader_election.cc:302] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:44.886277 22978 raft_consensus.cc:2932] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:44.887558 22715 raft_consensus.cc:2932] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:44.888327 22444 catalog_manager.cc:5484] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 4, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:44.893280 25331 raft_consensus.cc:2781] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:44.893334 25331 raft_consensus.cc:482] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:44.893369 25331 raft_consensus.cc:3037] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.894241 25331 raft_consensus.cc:504] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.894516 25331 leader_election.cc:288] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:44.895032 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4e687b2e36574856a45752104ce83953" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:44.895143 24871 raft_consensus.cc:3037] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.895922 24871 raft_consensus.cc:2454] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:44.896155 25199 leader_election.cc:302] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:44.896298 25331 raft_consensus.cc:2781] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:44.896363 25331 raft_consensus.cc:686] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:44.896519 25331 consensus_queue.cc:237] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:44.897159 25004 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4e687b2e36574856a45752104ce83953" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e" is_pre_election: true
I20240502 07:58:44.897294 25004 raft_consensus.cc:2454] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:44.897599 25003 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4e687b2e36574856a45752104ce83953" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e"
I20240502 07:58:44.897707 25003 raft_consensus.cc:3037] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:44.898232 22436 catalog_manager.cc:5484] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:44.901363 25003 raft_consensus.cc:2454] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:44.908987 22584 consensus_queue.cc:237] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:44.910527 22847 raft_consensus.cc:1261] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:44.911007 22522 consensus_peers.cc:485] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): Couldn't send request to peer 4654231dfe994c1a9d2e37d9c3d75699. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9ca2e5b5a0d24bcc9d1b69f998a0d928. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:44.911465 22978 raft_consensus.cc:1261] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:44.911795 26311 consensus_queue.cc:1035] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.912110 26311 consensus_queue.cc:1035] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:44.913805 26311 raft_consensus.cc:2932] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:44.914235 22978 raft_consensus.cc:2932] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:44.916769 22436 catalog_manager.cc:5484] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:44.920029 22847 raft_consensus.cc:2932] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:44.941247 24871 raft_consensus.cc:1203] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:44.955785 22444 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_10" } modify_external_catalogs: true
I20240502 07:58:44.955950 22444 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_10" } modify_external_catalogs: true
I20240502 07:58:44.958161 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 4 replicas of tablet 398405a3694c4d29b0627d2d74ebadc6
I20240502 07:58:44.958503 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 4 replicas of tablet 1e370c8fe68b4392ba3720533e538de3
I20240502 07:58:44.958650 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 398405a3694c4d29b0627d2d74ebadc6 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:44.958809 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 0e01743578f947cca1db2baec4fdee12
I20240502 07:58:44.958817 25468 tablet_replica.cc:330] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:44.958957 25468 raft_consensus.cc:2229] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.959106 25468 raft_consensus.cc:2258] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.959208 22828 tablet_service.cc:1514] Processing DeleteTablet for tablet 1e370c8fe68b4392ba3720533e538de3 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:44.959257 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet 398405a3694c4d29b0627d2d74ebadc6 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:44.959342 25467 tablet_replica.cc:330] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:44.959462 25467 raft_consensus.cc:2229] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.959580 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet 1e370c8fe68b4392ba3720533e538de3 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:44.959600 25467 raft_consensus.cc:2258] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.959682 22564 tablet_service.cc:1514] Processing DeleteTablet for tablet 0e01743578f947cca1db2baec4fdee12 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:44.959729 26109 tablet_replica.cc:330] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:44.959836 26109 raft_consensus.cc:2229] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:44.959820 25114 tablet_service.cc:1514] Processing DeleteTablet for tablet 0e01743578f947cca1db2baec4fdee12 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:44.960048 26109 raft_consensus.cc:2258] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.960218 24984 tablet_service.cc:1514] Processing DeleteTablet for tablet 1e370c8fe68b4392ba3720533e538de3 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:42878
I20240502 07:58:44.960248 26369 tablet_replica.cc:330] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:44.960354 26369 raft_consensus.cc:2229] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.960470 26369 raft_consensus.cc:2258] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.960484 26642 tablet_replica.cc:330] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e: stopping tablet replica
I20240502 07:58:44.960628 26642 raft_consensus.cc:2229] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Raft consensus shutting down.
I20240502 07:58:44.960673 24850 tablet_service.cc:1514] Processing DeleteTablet for tablet 398405a3694c4d29b0627d2d74ebadc6 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:44.960784 26642 raft_consensus.cc:2258] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Raft consensus is shut down!
I20240502 07:58:44.960845 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet 398405a3694c4d29b0627d2d74ebadc6 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:44.961009 25465 tablet_replica.cc:330] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:44.961059 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet 0e01743578f947cca1db2baec4fdee12 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:44.961120 25465 raft_consensus.cc:2229] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.961215 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 1e370c8fe68b4392ba3720533e538de3 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:44 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:44.961247 25465 raft_consensus.cc:2258] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.961359 26642 ts_tablet_manager.cc:1875] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.961545 26369 ts_tablet_manager.cc:1875] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
W20240502 07:58:44.961586 22915 consensus_peers.cc:485] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): Couldn't send request to peer 7db4178eed174225bfb24f5f9445ce88. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: STOPPING. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:44.962293 26109 ts_tablet_manager.cc:1875] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.962395 25465 ts_tablet_manager.cc:1875] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.963435 26642 ts_tablet_manager.cc:1888] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:44.963452 26369 ts_tablet_manager.cc:1888] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:44.963501 26642 log.cc:1196] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/wals/1e370c8fe68b4392ba3720533e538de3
I20240502 07:58:44.963515 26369 log.cc:1196] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/0e01743578f947cca1db2baec4fdee12
I20240502 07:58:44.963783 26369 ts_tablet_manager.cc:1909] T 0e01743578f947cca1db2baec4fdee12 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:44.963785 26642 ts_tablet_manager.cc:1909] T 1e370c8fe68b4392ba3720533e538de3 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting consensus metadata
I20240502 07:58:44.964351 25467 ts_tablet_manager.cc:1875] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.964449 26109 ts_tablet_manager.cc:1888] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:44.964515 26109 log.cc:1196] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/398405a3694c4d29b0627d2d74ebadc6
I20240502 07:58:44.964808 22430 catalog_manager.cc:4830] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): tablet 1e370c8fe68b4392ba3720533e538de3 (table rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]) successfully deleted
I20240502 07:58:44.965049 26109 ts_tablet_manager.cc:1909] T 398405a3694c4d29b0627d2d74ebadc6 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:44.965247 25468 ts_tablet_manager.cc:1875] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.965520 26109 tablet_replica.cc:330] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:44.965663 26109 raft_consensus.cc:2229] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:44.965854 26109 raft_consensus.cc:2258] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.966202 25467 ts_tablet_manager.cc:1888] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:44.966259 25467 log.cc:1196] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/1e370c8fe68b4392ba3720533e538de3
I20240502 07:58:44.966399 26109 ts_tablet_manager.cc:1875] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.966460 25467 ts_tablet_manager.cc:1909] T 1e370c8fe68b4392ba3720533e538de3 P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:44.966919 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 398405a3694c4d29b0627d2d74ebadc6 (table rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]) successfully deleted
I20240502 07:58:44.966992 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 1e370c8fe68b4392ba3720533e538de3 (table rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]) successfully deleted
I20240502 07:58:44.967247 25468 ts_tablet_manager.cc:1888] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:44.967319 25468 log.cc:1196] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/398405a3694c4d29b0627d2d74ebadc6
I20240502 07:58:44.967518 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet 0e01743578f947cca1db2baec4fdee12 (table rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]) successfully deleted
I20240502 07:58:44.967568 25468 ts_tablet_manager.cc:1909] T 398405a3694c4d29b0627d2d74ebadc6 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:44.968062 26109 ts_tablet_manager.cc:1888] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:44.968107 25468 tablet_replica.cc:330] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:44.968125 26109 log.cc:1196] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/1e370c8fe68b4392ba3720533e538de3
I20240502 07:58:44.968149 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 398405a3694c4d29b0627d2d74ebadc6 (table rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]) successfully deleted
I20240502 07:58:44.968228 25468 raft_consensus.cc:2229] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.968369 26109 ts_tablet_manager.cc:1909] T 1e370c8fe68b4392ba3720533e538de3 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:44.968377 25468 raft_consensus.cc:2258] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.968889 25468 ts_tablet_manager.cc:1875] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.964625 25465 ts_tablet_manager.cc:1888] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:44.969094 25465 log.cc:1196] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/398405a3694c4d29b0627d2d74ebadc6
I20240502 07:58:44.969372 25465 ts_tablet_manager.cc:1909] T 398405a3694c4d29b0627d2d74ebadc6 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:44.969540 26643 tablet_replica.cc:330] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:44.969698 26643 raft_consensus.cc:2229] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Raft consensus shutting down.
I20240502 07:58:44.969713 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 1e370c8fe68b4392ba3720533e538de3 (table rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]) successfully deleted
I20240502 07:58:44.969930 25465 tablet_replica.cc:330] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:44.970045 25465 raft_consensus.cc:2229] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:44.970214 25465 raft_consensus.cc:2258] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.970218 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 398405a3694c4d29b0627d2d74ebadc6 (table rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]) successfully deleted
I20240502 07:58:44.970322 26643 raft_consensus.cc:2258] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Raft consensus is shut down!
I20240502 07:58:44.970547 25468 ts_tablet_manager.cc:1888] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:44.970606 25468 log.cc:1196] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/0e01743578f947cca1db2baec4fdee12
I20240502 07:58:44.970844 25468 ts_tablet_manager.cc:1909] T 0e01743578f947cca1db2baec4fdee12 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:44.970929 25465 ts_tablet_manager.cc:1875] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.971302 25468 tablet_replica.cc:330] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:44.971371 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 0e01743578f947cca1db2baec4fdee12 (table rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]) successfully deleted
I20240502 07:58:44.971413 25468 raft_consensus.cc:2229] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:44.971537 25468 raft_consensus.cc:2258] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:44.972009 25468 ts_tablet_manager.cc:1875] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.973645 25468 ts_tablet_manager.cc:1888] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:44.973749 25468 log.cc:1196] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/1e370c8fe68b4392ba3720533e538de3
I20240502 07:58:44.973739 25465 ts_tablet_manager.cc:1888] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:44.973814 25465 log.cc:1196] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/0e01743578f947cca1db2baec4fdee12
I20240502 07:58:44.973924 26643 ts_tablet_manager.cc:1875] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:44.974074 25468 ts_tablet_manager.cc:1909] T 1e370c8fe68b4392ba3720533e538de3 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:44.974200 25465 ts_tablet_manager.cc:1909] T 0e01743578f947cca1db2baec4fdee12 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:44.974798 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 0e01743578f947cca1db2baec4fdee12 (table rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]) successfully deleted
I20240502 07:58:44.974993 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 1e370c8fe68b4392ba3720533e538de3 (table rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]) successfully deleted
I20240502 07:58:44.976891 26643 ts_tablet_manager.cc:1888] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:44.976943 26643 log.cc:1196] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/398405a3694c4d29b0627d2d74ebadc6
I20240502 07:58:44.977209 26643 ts_tablet_manager.cc:1909] T 398405a3694c4d29b0627d2d74ebadc6 P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:44.977911 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet 398405a3694c4d29b0627d2d74ebadc6 (table rebalance_test_table_10 [id=83ae5faa858c4513832a0c3fecc829d8]) successfully deleted
I20240502 07:58:45.028438 25003 consensus_queue.cc:237] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 07:58:45.029942 25265 raft_consensus.cc:1261] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:45.030735 25134 raft_consensus.cc:1261] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:45.031457 26696 consensus_queue.cc:1035] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:45.031656 26697 consensus_queue.cc:1035] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:45.033645 25493 raft_consensus.cc:2932] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
W20240502 07:58:45.033879 24939 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e -> Peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): Couldn't send request to peer 7db4178eed174225bfb24f5f9445ce88. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 28832d9e18ea41b194cf33d8c7971e23. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:45.034224 25265 raft_consensus.cc:2932] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:45.034268 25134 raft_consensus.cc:2932] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:45.036415 22444 catalog_manager.cc:5484] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New cstate: current_term: 1 leader_uuid: "74f9aa91c2c448f980b512fe2e8d672e" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:45.090200 25265 raft_consensus.cc:1203] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:45.095220 25874 raft_consensus.cc:482] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:45.095342 25874 raft_consensus.cc:504] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.095824 25874 leader_election.cc:288] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:45.096457 25003 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "92d6b875b1a942dc94cbf54dd42fd2f6" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e" is_pre_election: true
I20240502 07:58:45.096627 25003 raft_consensus.cc:2454] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:45.096974 25070 leader_election.cc:302] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:45.097132 25874 raft_consensus.cc:2781] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:45.097179 25874 raft_consensus.cc:482] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:45.097216 25874 raft_consensus.cc:3037] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:45.098238 25874 raft_consensus.cc:504] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.098526 25874 leader_election.cc:288] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:45.100574 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "92d6b875b1a942dc94cbf54dd42fd2f6" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:45.100689 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "92d6b875b1a942dc94cbf54dd42fd2f6" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:45.100729 25265 raft_consensus.cc:2454] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:45.100862 25003 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "92d6b875b1a942dc94cbf54dd42fd2f6" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e"
I20240502 07:58:45.100975 25003 raft_consensus.cc:3037] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:45.101846 25003 raft_consensus.cc:2454] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:45.102102 25070 leader_election.cc:302] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 74f9aa91c2c448f980b512fe2e8d672e; no voters: 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:45.102268 25874 raft_consensus.cc:2781] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:45.102454 25874 raft_consensus.cc:686] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:58:45.102631 25874 consensus_queue.cc:237] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.104369 22440 catalog_manager.cc:5484] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:45.111769 26413 ts_tablet_manager.cc:902] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69: Initiating tablet copy from peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:45.112258 26413 tablet_copy_client.cc:308] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:37451
I20240502 07:58:45.115859 22606 tablet_copy_service.cc:140] P 8bea0fe8d8044a2685f92f7a10c9cf85: Received BeginTabletCopySession request for tablet c53e8b3bd774466dbafe4994472975bc from peer 7eced220cf1642f3873fc385b6191e69 ({username='slave'} at 127.16.85.136:57807)
I20240502 07:58:45.115940 22606 tablet_copy_service.cc:161] P 8bea0fe8d8044a2685f92f7a10c9cf85: Beginning new tablet copy session on tablet c53e8b3bd774466dbafe4994472975bc from peer 7eced220cf1642f3873fc385b6191e69 at {username='slave'} at 127.16.85.136:57807: session id = 7eced220cf1642f3873fc385b6191e69-c53e8b3bd774466dbafe4994472975bc
I20240502 07:58:45.116572 22606 tablet_copy_source_session.cc:205] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:45.117319 26413 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c53e8b3bd774466dbafe4994472975bc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.118813 26413 tablet_copy_client.cc:783] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:45.118949 26413 tablet_copy_client.cc:647] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:45.120121 26413 tablet_copy_client.cc:523] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:45.121011 26413 tablet_bootstrap.cc:492] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:45.136655 26413 tablet_bootstrap.cc:492] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:45.136938 26413 tablet_bootstrap.cc:492] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69: Bootstrap complete.
I20240502 07:58:45.137084 26413 ts_tablet_manager.cc:1367] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.016s user 0.009s sys 0.006s
I20240502 07:58:45.137420 26413 raft_consensus.cc:348] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:45.137518 26413 raft_consensus.cc:729] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: LEARNER
I20240502 07:58:45.137607 26413 consensus_queue.cc:260] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:45.137877 26413 ts_tablet_manager.cc:1398] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:45.138298 22606 tablet_copy_service.cc:342] P 8bea0fe8d8044a2685f92f7a10c9cf85: Request end of tablet copy session 7eced220cf1642f3873fc385b6191e69-c53e8b3bd774466dbafe4994472975bc received from {username='slave'} at 127.16.85.136:57807
I20240502 07:58:45.138360 22606 tablet_copy_service.cc:433] P 8bea0fe8d8044a2685f92f7a10c9cf85: ending tablet copy session 7eced220cf1642f3873fc385b6191e69-c53e8b3bd774466dbafe4994472975bc on tablet c53e8b3bd774466dbafe4994472975bc with peer 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:45.213776 26413 ts_tablet_manager.cc:902] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:45.214491 26413 tablet_copy_client.cc:308] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:45.215140 22998 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet ca85b40df34d4e1c91a55d7dd69e1494 from peer 7eced220cf1642f3873fc385b6191e69 ({username='slave'} at 127.16.85.136:40185)
I20240502 07:58:45.215260 22998 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet ca85b40df34d4e1c91a55d7dd69e1494 from peer 7eced220cf1642f3873fc385b6191e69 at {username='slave'} at 127.16.85.136:40185: session id = 7eced220cf1642f3873fc385b6191e69-ca85b40df34d4e1c91a55d7dd69e1494
I20240502 07:58:45.216290 22998 tablet_copy_source_session.cc:205] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:45.217119 26413 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ca85b40df34d4e1c91a55d7dd69e1494. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.220921 26413 tablet_copy_client.cc:783] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:45.221135 26413 tablet_copy_client.cc:647] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:45.222194 26413 tablet_copy_client.cc:523] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:45.223201 26413 tablet_bootstrap.cc:492] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:45.225227 26284 consensus_queue.cc:1035] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:45.229586 26079 consensus_queue.cc:1035] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:45.241919 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:45.243916 26413 tablet_bootstrap.cc:492] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:45.244104 26791 ts_tablet_manager.cc:902] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e: Initiating tablet copy from peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:45.244628 26413 tablet_bootstrap.cc:492] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69: Bootstrap complete.
I20240502 07:58:45.244670 26791 tablet_copy_client.cc:308] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:37451
I20240502 07:58:45.244915 26413 ts_tablet_manager.cc:1367] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.022s user 0.016s sys 0.005s
I20240502 07:58:45.245517 22606 tablet_copy_service.cc:140] P 8bea0fe8d8044a2685f92f7a10c9cf85: Received BeginTabletCopySession request for tablet b583c8775a874ad1a89c82c00c05bbbd from peer 74f9aa91c2c448f980b512fe2e8d672e ({username='slave'} at 127.16.85.134:42087)
I20240502 07:58:45.245628 22606 tablet_copy_service.cc:161] P 8bea0fe8d8044a2685f92f7a10c9cf85: Beginning new tablet copy session on tablet b583c8775a874ad1a89c82c00c05bbbd from peer 74f9aa91c2c448f980b512fe2e8d672e at {username='slave'} at 127.16.85.134:42087: session id = 74f9aa91c2c448f980b512fe2e8d672e-b583c8775a874ad1a89c82c00c05bbbd
I20240502 07:58:45.245791 26413 raft_consensus.cc:348] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:45.245915 26413 raft_consensus.cc:729] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: LEARNER
I20240502 07:58:45.245990 26413 consensus_queue.cc:260] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:45.246243 26413 ts_tablet_manager.cc:1398] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:45.246479 22606 tablet_copy_source_session.cc:205] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:45.246965 25265 raft_consensus.cc:3037] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:45.247051 25003 raft_consensus.cc:1261] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:45.247471 25874 consensus_queue.cc:1035] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:45.247678 22998 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 7eced220cf1642f3873fc385b6191e69-ca85b40df34d4e1c91a55d7dd69e1494 received from {username='slave'} at 127.16.85.136:40185
I20240502 07:58:45.247757 22998 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 7eced220cf1642f3873fc385b6191e69-ca85b40df34d4e1c91a55d7dd69e1494 on tablet ca85b40df34d4e1c91a55d7dd69e1494 with peer 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:45.247912 25265 raft_consensus.cc:1261] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:45.248374 26791 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b583c8775a874ad1a89c82c00c05bbbd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.248311 26764 consensus_queue.cc:1035] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:45.250095 22440 catalog_manager.cc:5576] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857) does not have the latest schema for tablet 92d6b875b1a942dc94cbf54dd42fd2f6 (table rebalancer_dynamic_table [id=2f175365958540dbb77eef3ec4df49bd]). Expected version 1 got 0
I20240502 07:58:45.251407 22440 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet 92d6b875b1a942dc94cbf54dd42fd2f6 (table rebalancer_dynamic_table [id=2f175365958540dbb77eef3ec4df49bd]). Expected version 1 got 0
I20240502 07:58:45.251498 26791 tablet_copy_client.cc:783] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:45.251699 26791 tablet_copy_client.cc:647] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:45.251884 22436 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 92d6b875b1a942dc94cbf54dd42fd2f6 (table rebalancer_dynamic_table [id=2f175365958540dbb77eef3ec4df49bd]). Expected version 1 got 0
I20240502 07:58:45.252142 26800 tablet.cc:1722] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:45.252390 26800 tablet.cc:1620] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e: MemRowSet was empty: no flush needed.
I20240502 07:58:45.252887 26791 tablet_copy_client.cc:523] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:45.253615 26799 tablet.cc:1722] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:45.253908 26799 tablet.cc:1620] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:45.254019 26791 tablet_bootstrap.cc:492] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:45.255373 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=2f175365958540dbb77eef3ec4df49bd] alter complete (version 1)
I20240502 07:58:45.256525 26802 tablet.cc:1722] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:45.256973 26802 tablet.cc:1620] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:45.273756 26791 tablet_bootstrap.cc:492] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:45.273991 26791 tablet_bootstrap.cc:492] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap complete.
I20240502 07:58:45.274142 26791 ts_tablet_manager.cc:1367] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.020s user 0.014s sys 0.003s
I20240502 07:58:45.274529 26791 raft_consensus.cc:348] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:45.274622 26791 raft_consensus.cc:729] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: LEARNER
I20240502 07:58:45.274708 26791 consensus_queue.cc:260] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:45.274937 26791 ts_tablet_manager.cc:1398] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:45.275396 22606 tablet_copy_service.cc:342] P 8bea0fe8d8044a2685f92f7a10c9cf85: Request end of tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-b583c8775a874ad1a89c82c00c05bbbd received from {username='slave'} at 127.16.85.134:42087
I20240502 07:58:45.275470 22606 tablet_copy_service.cc:433] P 8bea0fe8d8044a2685f92f7a10c9cf85: ending tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-b583c8775a874ad1a89c82c00c05bbbd on tablet b583c8775a874ad1a89c82c00c05bbbd with peer 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:45.289624 26758 consensus_queue.cc:1035] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:45.293107 26758 consensus_queue.cc:1035] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:45.293769 26791 ts_tablet_manager.cc:902] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:45.294351 26791 tablet_copy_client.cc:308] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33387
I20240502 07:58:45.294832 22737 tablet_copy_service.cc:140] P 725969d490a04fdfb3ba890c3f5096e3: Received BeginTabletCopySession request for tablet 539cf5d4596f481cb03ab4d13a19198c from peer 74f9aa91c2c448f980b512fe2e8d672e ({username='slave'} at 127.16.85.134:53317)
I20240502 07:58:45.294934 22737 tablet_copy_service.cc:161] P 725969d490a04fdfb3ba890c3f5096e3: Beginning new tablet copy session on tablet 539cf5d4596f481cb03ab4d13a19198c from peer 74f9aa91c2c448f980b512fe2e8d672e at {username='slave'} at 127.16.85.134:53317: session id = 74f9aa91c2c448f980b512fe2e8d672e-539cf5d4596f481cb03ab4d13a19198c
I20240502 07:58:45.295840 22737 tablet_copy_source_session.cc:205] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:45.296751 26791 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 539cf5d4596f481cb03ab4d13a19198c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.298308 26791 tablet_copy_client.cc:783] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:45.298431 26791 tablet_copy_client.cc:647] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:45.299490 26791 tablet_copy_client.cc:523] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:45.300321 26791 tablet_bootstrap.cc:492] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:45.307582 26392 ts_tablet_manager.cc:902] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537: Initiating tablet copy from peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:45.308001 26392 tablet_copy_client.cc:308] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:37451
I20240502 07:58:45.308400 22606 tablet_copy_service.cc:140] P 8bea0fe8d8044a2685f92f7a10c9cf85: Received BeginTabletCopySession request for tablet 887c4c76980d4672aec02b2dbd4d8c89 from peer 6d02017318a2461796d431962fe6a537 ({username='slave'} at 127.16.85.133:57895)
I20240502 07:58:45.308708 22606 tablet_copy_service.cc:161] P 8bea0fe8d8044a2685f92f7a10c9cf85: Beginning new tablet copy session on tablet 887c4c76980d4672aec02b2dbd4d8c89 from peer 6d02017318a2461796d431962fe6a537 at {username='slave'} at 127.16.85.133:57895: session id = 6d02017318a2461796d431962fe6a537-887c4c76980d4672aec02b2dbd4d8c89
I20240502 07:58:45.309361 22606 tablet_copy_source_session.cc:205] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:45.310072 26392 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 887c4c76980d4672aec02b2dbd4d8c89. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.311858 26392 tablet_copy_client.cc:783] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:45.312000 26392 tablet_copy_client.cc:647] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:45.313170 26392 tablet_copy_client.cc:523] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:45.314311 26392 tablet_bootstrap.cc:492] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:45.318840 26791 tablet_bootstrap.cc:492] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:45.319034 26791 tablet_bootstrap.cc:492] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap complete.
I20240502 07:58:45.319165 26791 ts_tablet_manager.cc:1367] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.019s user 0.017s sys 0.000s
I20240502 07:58:45.319574 26791 raft_consensus.cc:348] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:45.319937 26791 raft_consensus.cc:729] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: LEARNER
I20240502 07:58:45.320071 26791 consensus_queue.cc:260] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:45.320410 26791 ts_tablet_manager.cc:1398] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.003s sys 0.000s
I20240502 07:58:45.320441 26817 ts_tablet_manager.cc:902] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699: Initiating tablet copy from peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:45.320953 22737 tablet_copy_service.cc:342] P 725969d490a04fdfb3ba890c3f5096e3: Request end of tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-539cf5d4596f481cb03ab4d13a19198c received from {username='slave'} at 127.16.85.134:53317
I20240502 07:58:45.321025 22737 tablet_copy_service.cc:433] P 725969d490a04fdfb3ba890c3f5096e3: ending tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-539cf5d4596f481cb03ab4d13a19198c on tablet 539cf5d4596f481cb03ab4d13a19198c with peer 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:45.321724 26817 tablet_copy_client.cc:308] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:37451
I20240502 07:58:45.322526 22606 tablet_copy_service.cc:140] P 8bea0fe8d8044a2685f92f7a10c9cf85: Received BeginTabletCopySession request for tablet 9ca2e5b5a0d24bcc9d1b69f998a0d928 from peer 4654231dfe994c1a9d2e37d9c3d75699 ({username='slave'} at 127.16.85.135:41701)
I20240502 07:58:45.322615 22606 tablet_copy_service.cc:161] P 8bea0fe8d8044a2685f92f7a10c9cf85: Beginning new tablet copy session on tablet 9ca2e5b5a0d24bcc9d1b69f998a0d928 from peer 4654231dfe994c1a9d2e37d9c3d75699 at {username='slave'} at 127.16.85.135:41701: session id = 4654231dfe994c1a9d2e37d9c3d75699-9ca2e5b5a0d24bcc9d1b69f998a0d928
I20240502 07:58:45.323421 22606 tablet_copy_source_session.cc:205] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:45.324254 26817 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9ca2e5b5a0d24bcc9d1b69f998a0d928. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.324945 26827 ts_tablet_manager.cc:902] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699: Initiating tablet copy from peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:45.325425 26827 tablet_copy_client.cc:308] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.131:44055
I20240502 07:58:45.326633 26817 tablet_copy_client.cc:783] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:45.326772 26817 tablet_copy_client.cc:647] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:45.330858 22868 tablet_copy_service.cc:140] P 7db4178eed174225bfb24f5f9445ce88: Received BeginTabletCopySession request for tablet 015ed33d8c634473997eaa60e91269a2 from peer 4654231dfe994c1a9d2e37d9c3d75699 ({username='slave'} at 127.16.85.135:47281)
I20240502 07:58:45.330947 22868 tablet_copy_service.cc:161] P 7db4178eed174225bfb24f5f9445ce88: Beginning new tablet copy session on tablet 015ed33d8c634473997eaa60e91269a2 from peer 4654231dfe994c1a9d2e37d9c3d75699 at {username='slave'} at 127.16.85.135:47281: session id = 4654231dfe994c1a9d2e37d9c3d75699-015ed33d8c634473997eaa60e91269a2
I20240502 07:58:45.331734 22868 tablet_copy_source_session.cc:205] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:45.332113 26817 tablet_copy_client.cc:523] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:45.332432 26827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 015ed33d8c634473997eaa60e91269a2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.333223 26817 tablet_bootstrap.cc:492] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:45.334322 26827 tablet_copy_client.cc:783] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:45.334455 26827 tablet_copy_client.cc:647] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:45.335386 26827 tablet_copy_client.cc:523] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:45.336266 26827 tablet_bootstrap.cc:492] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:45.339262 26392 tablet_bootstrap.cc:492] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:45.339524 26392 tablet_bootstrap.cc:492] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537: Bootstrap complete.
I20240502 07:58:45.339674 26392 ts_tablet_manager.cc:1367] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.025s user 0.019s sys 0.000s
I20240502 07:58:45.340056 26392 raft_consensus.cc:348] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:45.340216 26392 raft_consensus.cc:729] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: LEARNER
I20240502 07:58:45.340354 26392 consensus_queue.cc:260] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:45.340759 26392 ts_tablet_manager.cc:1398] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.004s sys 0.000s
I20240502 07:58:45.341375 22606 tablet_copy_service.cc:342] P 8bea0fe8d8044a2685f92f7a10c9cf85: Request end of tablet copy session 6d02017318a2461796d431962fe6a537-887c4c76980d4672aec02b2dbd4d8c89 received from {username='slave'} at 127.16.85.133:57895
I20240502 07:58:45.341445 22606 tablet_copy_service.cc:433] P 8bea0fe8d8044a2685f92f7a10c9cf85: ending tablet copy session 6d02017318a2461796d431962fe6a537-887c4c76980d4672aec02b2dbd4d8c89 on tablet 887c4c76980d4672aec02b2dbd4d8c89 with peer 6d02017318a2461796d431962fe6a537
I20240502 07:58:45.344343 26392 ts_tablet_manager.cc:902] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:45.344717 26392 tablet_copy_client.cc:308] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:45.345062 22998 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet e5d5f67e026344998d5fb079f532d076 from peer 6d02017318a2461796d431962fe6a537 ({username='slave'} at 127.16.85.133:38655)
I20240502 07:58:45.345147 22998 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet e5d5f67e026344998d5fb079f532d076 from peer 6d02017318a2461796d431962fe6a537 at {username='slave'} at 127.16.85.133:38655: session id = 6d02017318a2461796d431962fe6a537-e5d5f67e026344998d5fb079f532d076
I20240502 07:58:45.345963 22998 tablet_copy_source_session.cc:205] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:45.346627 26392 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e5d5f67e026344998d5fb079f532d076. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.348747 26392 tablet_copy_client.cc:783] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:45.348904 26392 tablet_copy_client.cc:647] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:45.350054 26392 tablet_copy_client.cc:523] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:45.351197 26392 tablet_bootstrap.cc:492] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:45.353595 26817 tablet_bootstrap.cc:492] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:45.353878 26817 tablet_bootstrap.cc:492] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap complete.
I20240502 07:58:45.354071 26817 ts_tablet_manager.cc:1367] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.021s user 0.019s sys 0.002s
I20240502 07:58:45.354442 26817 raft_consensus.cc:348] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:45.354606 26817 raft_consensus.cc:729] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: LEARNER
I20240502 07:58:45.354746 26817 consensus_queue.cc:260] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:45.355229 26817 ts_tablet_manager.cc:1398] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:45.355644 22606 tablet_copy_service.cc:342] P 8bea0fe8d8044a2685f92f7a10c9cf85: Request end of tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-9ca2e5b5a0d24bcc9d1b69f998a0d928 received from {username='slave'} at 127.16.85.135:41701
I20240502 07:58:45.355716 22606 tablet_copy_service.cc:433] P 8bea0fe8d8044a2685f92f7a10c9cf85: ending tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-9ca2e5b5a0d24bcc9d1b69f998a0d928 on tablet 9ca2e5b5a0d24bcc9d1b69f998a0d928 with peer 4654231dfe994c1a9d2e37d9c3d75699
I20240502 07:58:45.359325 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:45.363124 26128 raft_consensus.cc:1050] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 6d02017318a2461796d431962fe6a537 to VOTER
I20240502 07:58:45.363502 26128 consensus_queue.cc:237] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:45.365550 22847 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:45.365782 22716 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:45.366045 24871 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:45.366289 25669 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:45.367173 25670 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:45.367427 25670 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:45.373160 26799 tablet.cc:1722] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:45.373474 26799 tablet.cc:1620] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:45.374372 26802 tablet.cc:1722] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:45.374624 26802 tablet.cc:1620] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:45.375190 25669 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:45.376564 26392 tablet_bootstrap.cc:492] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:45.376868 26392 tablet_bootstrap.cc:492] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537: Bootstrap complete.
I20240502 07:58:45.377076 26392 ts_tablet_manager.cc:1367] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.026s user 0.025s sys 0.000s
I20240502 07:58:45.377494 26392 raft_consensus.cc:348] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:45.377705 26392 raft_consensus.cc:729] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: LEARNER
I20240502 07:58:45.377857 26392 consensus_queue.cc:260] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:45.379235 26392 ts_tablet_manager.cc:1398] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:45.380064 22998 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 6d02017318a2461796d431962fe6a537-e5d5f67e026344998d5fb079f532d076 received from {username='slave'} at 127.16.85.133:38655
I20240502 07:58:45.380131 22998 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 6d02017318a2461796d431962fe6a537-e5d5f67e026344998d5fb079f532d076 on tablet e5d5f67e026344998d5fb079f532d076 with peer 6d02017318a2461796d431962fe6a537
I20240502 07:58:45.380944 22435 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=2f175365958540dbb77eef3ec4df49bd] alter complete (version 2)
I20240502 07:58:45.380867 22716 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:45.383577 22435 catalog_manager.cc:5484] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:45.383945 26800 tablet.cc:1722] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:45.384220 26800 tablet.cc:1620] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e: MemRowSet was empty: no flush needed.
I20240502 07:58:45.385679 24871 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:45.386152 22847 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:45.394542 26827 tablet_bootstrap.cc:492] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:45.394852 26827 tablet_bootstrap.cc:492] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap complete.
I20240502 07:58:45.395002 26827 ts_tablet_manager.cc:1367] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.059s user 0.023s sys 0.001s
I20240502 07:58:45.395439 26827 raft_consensus.cc:348] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:45.395558 26827 raft_consensus.cc:729] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: LEARNER
I20240502 07:58:45.395654 26827 consensus_queue.cc:260] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:45.395896 26827 ts_tablet_manager.cc:1398] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:45.396456 22868 tablet_copy_service.cc:342] P 7db4178eed174225bfb24f5f9445ce88: Request end of tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-015ed33d8c634473997eaa60e91269a2 received from {username='slave'} at 127.16.85.135:47281
I20240502 07:58:45.396528 22868 tablet_copy_service.cc:433] P 7db4178eed174225bfb24f5f9445ce88: ending tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-015ed33d8c634473997eaa60e91269a2 on tablet 015ed33d8c634473997eaa60e91269a2 with peer 4654231dfe994c1a9d2e37d9c3d75699
I20240502 07:58:45.417130 25003 raft_consensus.cc:3037] T 07409f4ddd0040fc98e6b2064e2df523 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:45.417296 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_8"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:45.418579 26079 consensus_queue.cc:1035] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:45.423597 24984 tablet_service.cc:1467] Processing CreateTablet for tablet 325b6a98112d4eef9488d7ef657bc375 (DEFAULT_TABLE table=rebalancer_extra_table_8 [id=3eca1e0de44446a9b1e8bb318b02389a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:45.423866 24984 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 325b6a98112d4eef9488d7ef657bc375. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.423952 25114 tablet_service.cc:1467] Processing CreateTablet for tablet 325b6a98112d4eef9488d7ef657bc375 (DEFAULT_TABLE table=rebalancer_extra_table_8 [id=3eca1e0de44446a9b1e8bb318b02389a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:45.424247 25114 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 325b6a98112d4eef9488d7ef657bc375. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.424322 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 325b6a98112d4eef9488d7ef657bc375 (DEFAULT_TABLE table=rebalancer_extra_table_8 [id=3eca1e0de44446a9b1e8bb318b02389a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:45.424557 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 325b6a98112d4eef9488d7ef657bc375. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.426774 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 6af67c65672f45a68e960936351e223f (DEFAULT_TABLE table=rebalancer_extra_table_8 [id=3eca1e0de44446a9b1e8bb318b02389a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:45.426939 24984 tablet_service.cc:1467] Processing CreateTablet for tablet 90acd69ebaba4fb2a67e16172e8d7aa5 (DEFAULT_TABLE table=rebalancer_extra_table_8 [id=3eca1e0de44446a9b1e8bb318b02389a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:45.427029 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6af67c65672f45a68e960936351e223f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.427012 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 6af67c65672f45a68e960936351e223f (DEFAULT_TABLE table=rebalancer_extra_table_8 [id=3eca1e0de44446a9b1e8bb318b02389a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:45.427121 24984 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 90acd69ebaba4fb2a67e16172e8d7aa5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.427244 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6af67c65672f45a68e960936351e223f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.428954 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 90acd69ebaba4fb2a67e16172e8d7aa5 (DEFAULT_TABLE table=rebalancer_extra_table_8 [id=3eca1e0de44446a9b1e8bb318b02389a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:45.429143 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 90acd69ebaba4fb2a67e16172e8d7aa5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.429246 26882 tablet_bootstrap.cc:492] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:45.430183 26882 tablet_bootstrap.cc:654] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:45.431603 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 6af67c65672f45a68e960936351e223f (DEFAULT_TABLE table=rebalancer_extra_table_8 [id=3eca1e0de44446a9b1e8bb318b02389a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:45.431789 25532 consensus_queue.cc:1035] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:45.431874 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6af67c65672f45a68e960936351e223f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.433763 26879 tablet_bootstrap.cc:492] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:45.435009 26880 tablet_bootstrap.cc:492] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:45.435513 26879 tablet_bootstrap.cc:654] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:45.437014 25114 tablet_service.cc:1467] Processing CreateTablet for tablet 90acd69ebaba4fb2a67e16172e8d7aa5 (DEFAULT_TABLE table=rebalancer_extra_table_8 [id=3eca1e0de44446a9b1e8bb318b02389a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:45.437222 25114 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 90acd69ebaba4fb2a67e16172e8d7aa5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.437259 26880 tablet_bootstrap.cc:654] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:45.438230 26880 tablet_bootstrap.cc:492] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:45.438318 26880 ts_tablet_manager.cc:1367] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:45.438730 26880 raft_consensus.cc:348] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.438824 26880 raft_consensus.cc:374] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:45.438851 26880 raft_consensus.cc:729] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:45.439010 26880 consensus_queue.cc:260] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.439738 26880 ts_tablet_manager.cc:1398] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:45.439906 26880 tablet_bootstrap.cc:492] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:45.441063 26880 tablet_bootstrap.cc:654] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:45.442448 26881 tablet_bootstrap.cc:492] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:45.443142 26885 tablet_bootstrap.cc:492] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:45.443948 26885 tablet_bootstrap.cc:654] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:45.444412 26886 tablet_bootstrap.cc:492] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:45.445333 26882 tablet_bootstrap.cc:492] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:45.445387 26881 tablet_bootstrap.cc:654] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:45.445828 26882 ts_tablet_manager.cc:1367] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.017s user 0.002s sys 0.000s
I20240502 07:58:45.446228 26882 raft_consensus.cc:348] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:45.446305 26882 raft_consensus.cc:374] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:45.446327 26882 raft_consensus.cc:729] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:45.446437 26886 tablet_bootstrap.cc:654] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:45.446422 26882 consensus_queue.cc:260] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:45.446648 26882 ts_tablet_manager.cc:1398] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:45.446774 26882 tablet_bootstrap.cc:492] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:45.447172 26885 tablet_bootstrap.cc:492] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:45.447230 26885 ts_tablet_manager.cc:1367] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:45.447538 26882 tablet_bootstrap.cc:654] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:45.447585 26885 raft_consensus.cc:348] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:45.447656 26885 raft_consensus.cc:374] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:45.447676 26885 raft_consensus.cc:729] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:45.447777 26885 consensus_queue.cc:260] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:45.448006 26885 ts_tablet_manager.cc:1398] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:45.448432 26882 tablet_bootstrap.cc:492] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:45.448431 26886 tablet_bootstrap.cc:492] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:45.448494 26886 ts_tablet_manager.cc:1367] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:58:45.448491 26882 ts_tablet_manager.cc:1367] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:45.449692 26886 raft_consensus.cc:348] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:45.449795 26886 raft_consensus.cc:374] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:45.449821 26886 raft_consensus.cc:729] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:45.449937 26886 consensus_queue.cc:260] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:45.450714 26886 ts_tablet_manager.cc:1398] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:45.451396 26881 tablet_bootstrap.cc:492] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:45.451332 26882 raft_consensus.cc:348] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.451478 26881 ts_tablet_manager.cc:1367] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.009s user 0.001s sys 0.000s
I20240502 07:58:45.451495 26882 raft_consensus.cc:374] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:45.451537 26882 raft_consensus.cc:729] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:45.451644 26882 consensus_queue.cc:260] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.452028 26882 ts_tablet_manager.cc:1398] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:45.452025 26881 raft_consensus.cc:348] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.452119 26881 raft_consensus.cc:374] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:45.452199 26881 raft_consensus.cc:729] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:45.452334 26881 consensus_queue.cc:260] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.454015 26827 ts_tablet_manager.cc:902] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:45.454414 26791 ts_tablet_manager.cc:902] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:45.454548 26827 tablet_copy_client.cc:308] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:45.454902 26791 tablet_copy_client.cc:308] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:45.454936 22998 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet e24e4e639b0649e3ac135ac9358a0398 from peer 4654231dfe994c1a9d2e37d9c3d75699 ({username='slave'} at 127.16.85.135:41731)
I20240502 07:58:45.455008 22998 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet e24e4e639b0649e3ac135ac9358a0398 from peer 4654231dfe994c1a9d2e37d9c3d75699 at {username='slave'} at 127.16.85.135:41731: session id = 4654231dfe994c1a9d2e37d9c3d75699-e24e4e639b0649e3ac135ac9358a0398
I20240502 07:58:45.455240 22999 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet eb0a7b1308114769b556795ae18a746e from peer 74f9aa91c2c448f980b512fe2e8d672e ({username='slave'} at 127.16.85.134:46615)
I20240502 07:58:45.455305 22999 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet eb0a7b1308114769b556795ae18a746e from peer 74f9aa91c2c448f980b512fe2e8d672e at {username='slave'} at 127.16.85.134:46615: session id = 74f9aa91c2c448f980b512fe2e8d672e-eb0a7b1308114769b556795ae18a746e
I20240502 07:58:45.455837 22998 tablet_copy_source_session.cc:205] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:45.455981 22999 tablet_copy_source_session.cc:205] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:45.456741 26881 ts_tablet_manager.cc:1398] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:45.456905 26827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e24e4e639b0649e3ac135ac9358a0398. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.456912 26791 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb0a7b1308114769b556795ae18a746e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.457571 26879 tablet_bootstrap.cc:492] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:45.457654 26879 ts_tablet_manager.cc:1367] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.024s user 0.002s sys 0.000s
I20240502 07:58:45.458088 26879 raft_consensus.cc:348] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.458204 26879 raft_consensus.cc:374] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:45.458233 26879 raft_consensus.cc:729] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:45.458357 26879 consensus_queue.cc:260] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.458609 26879 ts_tablet_manager.cc:1398] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:45.458700 26791 tablet_copy_client.cc:783] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:45.458808 26879 tablet_bootstrap.cc:492] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:45.458904 26827 tablet_copy_client.cc:783] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:45.458837 26791 tablet_copy_client.cc:647] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:45.459018 26827 tablet_copy_client.cc:647] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:45.459432 26880 tablet_bootstrap.cc:492] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:45.459506 26880 ts_tablet_manager.cc:1367] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.020s user 0.002s sys 0.000s
I20240502 07:58:45.459669 26879 tablet_bootstrap.cc:654] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:45.459880 26880 raft_consensus.cc:348] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.459964 26880 raft_consensus.cc:374] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:45.459992 26880 raft_consensus.cc:729] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:45.460083 26880 consensus_queue.cc:260] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.460289 26791 tablet_copy_client.cc:523] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:45.460326 26880 ts_tablet_manager.cc:1398] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:45.461668 26791 tablet_bootstrap.cc:492] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:45.462618 22440 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_12" } modify_external_catalogs: true
I20240502 07:58:45.462729 22440 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_12" } modify_external_catalogs: true
I20240502 07:58:45.464936 22440 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 90480cc47cac4c84994bfcb9bc8a2dbf
I20240502 07:58:45.465234 22440 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet f1830cb65a294734ad80f274b89ac185
I20240502 07:58:45.465440 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet 90480cc47cac4c84994bfcb9bc8a2dbf with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:45.465471 22440 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet eecae5d013f543bf8eed1f2488757115
I20240502 07:58:45.465881 25465 tablet_replica.cc:330] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:45.465874 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet f1830cb65a294734ad80f274b89ac185 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:45.466012 25465 raft_consensus.cc:2229] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:45.466054 26643 tablet_replica.cc:330] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:45.466066 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet f1830cb65a294734ad80f274b89ac185 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:45.466154 26413 ts_tablet_manager.cc:902] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69: Initiating tablet copy from peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:45.466176 26643 raft_consensus.cc:2229] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:45.466202 25465 raft_consensus.cc:2258] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.466306 26643 raft_consensus.cc:2258] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.466603 26413 tablet_copy_client.cc:308] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.131:44055
I20240502 07:58:45.466722 25465 ts_tablet_manager.cc:1875] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.467018 26643 ts_tablet_manager.cc:1875] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.467018 22868 tablet_copy_service.cc:140] P 7db4178eed174225bfb24f5f9445ce88: Received BeginTabletCopySession request for tablet abfc26b86b17464fa7cf0b13ab73861c from peer 7eced220cf1642f3873fc385b6191e69 ({username='slave'} at 127.16.85.136:46167)
I20240502 07:58:45.467106 22868 tablet_copy_service.cc:161] P 7db4178eed174225bfb24f5f9445ce88: Beginning new tablet copy session on tablet abfc26b86b17464fa7cf0b13ab73861c from peer 7eced220cf1642f3873fc385b6191e69 at {username='slave'} at 127.16.85.136:46167: session id = 7eced220cf1642f3873fc385b6191e69-abfc26b86b17464fa7cf0b13ab73861c
I20240502 07:58:45.467891 22868 tablet_copy_source_session.cc:205] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:45.468322 25465 ts_tablet_manager.cc:1888] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:45.468396 25465 log.cc:1196] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/90480cc47cac4c84994bfcb9bc8a2dbf
I20240502 07:58:45.468628 25465 ts_tablet_manager.cc:1909] T 90480cc47cac4c84994bfcb9bc8a2dbf P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:45.468745 26643 ts_tablet_manager.cc:1888] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:45.468793 26643 log.cc:1196] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/f1830cb65a294734ad80f274b89ac185
I20240502 07:58:45.468806 26413 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet abfc26b86b17464fa7cf0b13ab73861c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.468999 26643 ts_tablet_manager.cc:1909] T f1830cb65a294734ad80f274b89ac185 P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:45.470777 26413 tablet_copy_client.cc:783] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:45.470903 26413 tablet_copy_client.cc:647] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:45.471271 22564 tablet_service.cc:1514] Processing DeleteTablet for tablet eecae5d013f543bf8eed1f2488757115 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:45.471349 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet f1830cb65a294734ad80f274b89ac185 (table rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]) successfully deleted
I20240502 07:58:45.471449 25468 tablet_replica.cc:330] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:45.471534 25465 tablet_replica.cc:330] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:45.471576 25468 raft_consensus.cc:2229] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:45.471599 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet 90480cc47cac4c84994bfcb9bc8a2dbf with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:45.471652 25465 raft_consensus.cc:2229] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:45.471702 25468 raft_consensus.cc:2258] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.471796 25465 raft_consensus.cc:2258] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.471882 26918 tablet_replica.cc:330] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:45.471940 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet f1830cb65a294734ad80f274b89ac185 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:45.471987 26918 raft_consensus.cc:2229] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:45.472072 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet eecae5d013f543bf8eed1f2488757115 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:45.472110 26918 raft_consensus.cc:2258] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.472132 25468 ts_tablet_manager.cc:1875] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.472195 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 90480cc47cac4c84994bfcb9bc8a2dbf (table rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]) successfully deleted
I20240502 07:58:45.472448 22828 tablet_service.cc:1514] Processing DeleteTablet for tablet 90480cc47cac4c84994bfcb9bc8a2dbf with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:45.472512 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet eecae5d013f543bf8eed1f2488757115 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:45.472831 25465 ts_tablet_manager.cc:1875] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.472879 26918 ts_tablet_manager.cc:1875] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.474187 25465 ts_tablet_manager.cc:1888] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:45.474267 25465 log.cc:1196] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/f1830cb65a294734ad80f274b89ac185
I20240502 07:58:45.474260 25468 ts_tablet_manager.cc:1888] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:45.474321 25468 log.cc:1196] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/eecae5d013f543bf8eed1f2488757115
I20240502 07:58:45.474516 25465 ts_tablet_manager.cc:1909] T f1830cb65a294734ad80f274b89ac185 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:45.474565 25468 ts_tablet_manager.cc:1909] T eecae5d013f543bf8eed1f2488757115 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:45.474998 26918 ts_tablet_manager.cc:1888] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:45.475019 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet f1830cb65a294734ad80f274b89ac185 (table rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]) successfully deleted
I20240502 07:58:45.475055 26918 log.cc:1196] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/90480cc47cac4c84994bfcb9bc8a2dbf
I20240502 07:58:45.475091 25468 tablet_replica.cc:330] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:45.475247 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet eecae5d013f543bf8eed1f2488757115 (table rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]) successfully deleted
I20240502 07:58:45.475296 26918 ts_tablet_manager.cc:1909] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:58:45.475453 25468 raft_consensus.cc:2229] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:45.475559 26919 tablet_replica.cc:330] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:45.475658 25468 raft_consensus.cc:2258] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.475730 26919 raft_consensus.cc:2229] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:45.475848 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet 90480cc47cac4c84994bfcb9bc8a2dbf (table rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]) successfully deleted
I20240502 07:58:45.476019 26919 raft_consensus.cc:2258] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.476127 26920 tablet_replica.cc:330] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:45.476212 26413 tablet_copy_client.cc:523] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:45.476260 25468 ts_tablet_manager.cc:1875] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.476262 26920 raft_consensus.cc:2229] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:45.476454 26920 raft_consensus.cc:2258] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.476984 26920 ts_tablet_manager.cc:1875] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.477116 26919 ts_tablet_manager.cc:1875] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.477270 26413 tablet_bootstrap.cc:492] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:45.478066 25468 ts_tablet_manager.cc:1888] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:45.478132 25468 log.cc:1196] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/f1830cb65a294734ad80f274b89ac185
I20240502 07:58:45.478380 25468 ts_tablet_manager.cc:1909] T f1830cb65a294734ad80f274b89ac185 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:45.479396 26919 ts_tablet_manager.cc:1888] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:45.479446 26919 log.cc:1196] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/eecae5d013f543bf8eed1f2488757115
I20240502 07:58:45.479674 26919 ts_tablet_manager.cc:1909] T eecae5d013f543bf8eed1f2488757115 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:45.480077 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet f1830cb65a294734ad80f274b89ac185 (table rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]) successfully deleted
I20240502 07:58:45.480289 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet eecae5d013f543bf8eed1f2488757115 (table rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]) successfully deleted
I20240502 07:58:45.481532 26920 ts_tablet_manager.cc:1888] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:45.481659 26920 log.cc:1196] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/90480cc47cac4c84994bfcb9bc8a2dbf
I20240502 07:58:45.481911 26920 ts_tablet_manager.cc:1909] T 90480cc47cac4c84994bfcb9bc8a2dbf P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:45.482498 26920 tablet_replica.cc:330] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:45.482568 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 90480cc47cac4c84994bfcb9bc8a2dbf (table rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]) successfully deleted
I20240502 07:58:45.482694 26920 raft_consensus.cc:2229] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:45.482834 26920 raft_consensus.cc:2258] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.483337 26920 ts_tablet_manager.cc:1875] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.484891 26827 tablet_copy_client.cc:523] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:45.485074 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:45.486117 26827 tablet_bootstrap.cc:492] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:45.486378 26920 ts_tablet_manager.cc:1888] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:45.486500 26920 log.cc:1196] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/eecae5d013f543bf8eed1f2488757115
I20240502 07:58:45.487162 26920 ts_tablet_manager.cc:1909] T eecae5d013f543bf8eed1f2488757115 P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:45.489801 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet eecae5d013f543bf8eed1f2488757115 (table rebalance_test_table_12 [id=29407b46bb0a4941882fbada0f480038]) successfully deleted
I20240502 07:58:45.491235 26879 tablet_bootstrap.cc:492] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:45.491312 26879 ts_tablet_manager.cc:1367] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.033s user 0.002s sys 0.000s
I20240502 07:58:45.491729 26879 raft_consensus.cc:348] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.491824 26879 raft_consensus.cc:374] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:45.491852 26879 raft_consensus.cc:729] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:45.491964 26879 consensus_queue.cc:260] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.492200 26879 ts_tablet_manager.cc:1398] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:45.493010 26799 tablet.cc:1722] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:45.493517 26802 tablet.cc:1722] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:45.496804 26128 raft_consensus.cc:482] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:45.496897 26128 raft_consensus.cc:504] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:45.497040 26800 tablet.cc:1722] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:45.497252 26128 leader_election.cc:288] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:45.497502 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6af67c65672f45a68e960936351e223f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:45.497623 22584 raft_consensus.cc:2454] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:45.497949 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6af67c65672f45a68e960936351e223f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:45.498078 24871 raft_consensus.cc:2454] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:45.498304 22912 leader_election.cc:302] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:45.498534 25669 raft_consensus.cc:2781] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:45.498580 25669 raft_consensus.cc:482] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:45.498610 25669 raft_consensus.cc:3037] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:45.499429 25669 raft_consensus.cc:504] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:45.499720 25669 leader_election.cc:288] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:45.500365 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6af67c65672f45a68e960936351e223f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:45.500505 22584 raft_consensus.cc:3037] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:45.501425 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6af67c65672f45a68e960936351e223f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:45.501528 24871 raft_consensus.cc:3037] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:45.501616 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=2f175365958540dbb77eef3ec4df49bd] alter complete (version 3)
I20240502 07:58:45.502471 24871 raft_consensus.cc:2454] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:45.502761 22912 leader_election.cc:302] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:45.502903 26128 raft_consensus.cc:2781] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:45.502972 26128 raft_consensus.cc:686] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:45.503104 26128 consensus_queue.cc:237] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:45.505123 22436 catalog_manager.cc:5484] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:45.505419 22584 raft_consensus.cc:2454] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:45.506145 26791 tablet_bootstrap.cc:492] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:45.506472 26791 tablet_bootstrap.cc:492] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap complete.
I20240502 07:58:45.506693 26791 ts_tablet_manager.cc:1367] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.045s user 0.020s sys 0.003s
I20240502 07:58:45.507166 26791 raft_consensus.cc:348] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:45.507356 26791 raft_consensus.cc:729] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: LEARNER
I20240502 07:58:45.507476 26791 consensus_queue.cc:260] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:45.507905 26791 ts_tablet_manager.cc:1398] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:45.508291 22999 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-eb0a7b1308114769b556795ae18a746e received from {username='slave'} at 127.16.85.134:46615
I20240502 07:58:45.508354 22999 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-eb0a7b1308114769b556795ae18a746e on tablet eb0a7b1308114769b556795ae18a746e with peer 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:45.508404 26697 raft_consensus.cc:482] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:45.508467 26697 raft_consensus.cc:504] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.508859 26697 leader_election.cc:288] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:45.509215 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "90acd69ebaba4fb2a67e16172e8d7aa5" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:45.509328 24871 raft_consensus.cc:2454] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:45.510402 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "90acd69ebaba4fb2a67e16172e8d7aa5" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:45.510557 25134 raft_consensus.cc:2454] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:45.511025 24936 leader_election.cc:302] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:45.511432 26697 raft_consensus.cc:2781] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:45.511488 26697 raft_consensus.cc:482] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:45.511533 26697 raft_consensus.cc:3037] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:45.512459 26697 raft_consensus.cc:504] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.512765 26697 leader_election.cc:288] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:45.512815 22978 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "9d2283e5d28347519b38e876d840608e"
dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:53680
I20240502 07:58:45.512944 22978 raft_consensus.cc:595] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:45.513392 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "90acd69ebaba4fb2a67e16172e8d7aa5" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:45.513504 25134 raft_consensus.cc:3037] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:45.513567 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "90acd69ebaba4fb2a67e16172e8d7aa5" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:45.513664 24871 raft_consensus.cc:3037] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:45.514469 25134 raft_consensus.cc:2454] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:45.514493 24871 raft_consensus.cc:2454] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:45.514775 24939 leader_election.cc:302] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:45.515082 26697 raft_consensus.cc:2781] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:45.515152 26697 raft_consensus.cc:686] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Becoming Leader. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Running, Role: LEADER
I20240502 07:58:45.515259 26697 consensus_queue.cc:237] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.516865 22436 catalog_manager.cc:5484] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: term changed from 0 to 1, leader changed from <none> to 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134). New cstate: current_term: 1 leader_uuid: "74f9aa91c2c448f980b512fe2e8d672e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:45.520525 26413 tablet_bootstrap.cc:492] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:45.520773 26413 tablet_bootstrap.cc:492] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69: Bootstrap complete.
I20240502 07:58:45.520926 26413 ts_tablet_manager.cc:1367] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.044s user 0.017s sys 0.007s
I20240502 07:58:45.521327 26413 raft_consensus.cc:348] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:45.521445 26413 raft_consensus.cc:729] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: LEARNER
I20240502 07:58:45.521541 26413 consensus_queue.cc:260] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:45.521788 26413 ts_tablet_manager.cc:1398] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:45.522243 22868 tablet_copy_service.cc:342] P 7db4178eed174225bfb24f5f9445ce88: Request end of tablet copy session 7eced220cf1642f3873fc385b6191e69-abfc26b86b17464fa7cf0b13ab73861c received from {username='slave'} at 127.16.85.136:46167
I20240502 07:58:45.522315 22868 tablet_copy_service.cc:433] P 7db4178eed174225bfb24f5f9445ce88: ending tablet copy session 7eced220cf1642f3873fc385b6191e69-abfc26b86b17464fa7cf0b13ab73861c on tablet abfc26b86b17464fa7cf0b13ab73861c with peer 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:45.532245 26827 tablet_bootstrap.cc:492] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:45.532476 26827 tablet_bootstrap.cc:492] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap complete.
I20240502 07:58:45.532620 26827 ts_tablet_manager.cc:1367] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.047s user 0.019s sys 0.000s
I20240502 07:58:45.533027 26827 raft_consensus.cc:348] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:45.533170 26827 raft_consensus.cc:729] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: LEARNER
I20240502 07:58:45.533293 26827 consensus_queue.cc:260] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:45.533569 26827 ts_tablet_manager.cc:1398] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:45.534051 22999 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-e24e4e639b0649e3ac135ac9358a0398 received from {username='slave'} at 127.16.85.135:41731
I20240502 07:58:45.534122 22999 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-e24e4e639b0649e3ac135ac9358a0398 on tablet e24e4e639b0649e3ac135ac9358a0398 with peer 4654231dfe994c1a9d2e37d9c3d75699
I20240502 07:58:45.553915 26950 ts_tablet_manager.cc:902] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88: Initiating tablet copy from peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:45.554566 26950 tablet_copy_client.cc:308] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.134:34857
I20240502 07:58:45.555161 25024 tablet_copy_service.cc:140] P 74f9aa91c2c448f980b512fe2e8d672e: Received BeginTabletCopySession request for tablet 28832d9e18ea41b194cf33d8c7971e23 from peer 7db4178eed174225bfb24f5f9445ce88 ({username='slave'} at 127.16.85.131:57145)
I20240502 07:58:45.555270 25024 tablet_copy_service.cc:161] P 74f9aa91c2c448f980b512fe2e8d672e: Beginning new tablet copy session on tablet 28832d9e18ea41b194cf33d8c7971e23 from peer 7db4178eed174225bfb24f5f9445ce88 at {username='slave'} at 127.16.85.131:57145: session id = 7db4178eed174225bfb24f5f9445ce88-28832d9e18ea41b194cf33d8c7971e23
I20240502 07:58:45.556406 25024 tablet_copy_source_session.cc:205] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:45.557535 26950 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 28832d9e18ea41b194cf33d8c7971e23. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.559111 26950 tablet_copy_client.cc:783] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:45.559259 26950 tablet_copy_client.cc:647] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:45.560586 26950 tablet_copy_client.cc:523] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:45.561652 26950 tablet_bootstrap.cc:492] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:45.565071 26950 tablet_bootstrap.cc:492] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:45.565304 26950 tablet_bootstrap.cc:492] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap complete.
I20240502 07:58:45.565440 26950 ts_tablet_manager.cc:1367] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.004s user 0.004s sys 0.000s
I20240502 07:58:45.565758 26950 raft_consensus.cc:348] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 07:58:45.565846 26950 raft_consensus.cc:729] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: LEARNER
I20240502 07:58:45.565930 26950 consensus_queue.cc:260] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 07:58:45.566144 26950 ts_tablet_manager.cc:1398] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:45.566735 25024 tablet_copy_service.cc:342] P 74f9aa91c2c448f980b512fe2e8d672e: Request end of tablet copy session 7db4178eed174225bfb24f5f9445ce88-28832d9e18ea41b194cf33d8c7971e23 received from {username='slave'} at 127.16.85.131:57145
I20240502 07:58:45.566805 25024 tablet_copy_service.cc:433] P 74f9aa91c2c448f980b512fe2e8d672e: ending tablet copy session 7db4178eed174225bfb24f5f9445ce88-28832d9e18ea41b194cf33d8c7971e23 on tablet 28832d9e18ea41b194cf33d8c7971e23 with peer 7db4178eed174225bfb24f5f9445ce88
I20240502 07:58:45.602699 22436 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:45.602831 22436 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:45.604672 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 92d6b875b1a942dc94cbf54dd42fd2f6
I20240502 07:58:45.605235 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet 92d6b875b1a942dc94cbf54dd42fd2f6 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:45.605245 24984 tablet_service.cc:1514] Processing DeleteTablet for tablet 92d6b875b1a942dc94cbf54dd42fd2f6 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:42878
I20240502 07:58:45.605409 25114 tablet_service.cc:1514] Processing DeleteTablet for tablet 92d6b875b1a942dc94cbf54dd42fd2f6 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:45.605540 26956 tablet_replica.cc:330] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e: stopping tablet replica
I20240502 07:58:45.605655 26918 tablet_replica.cc:330] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:45.605674 26956 raft_consensus.cc:2229] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:45.605691 26957 tablet_replica.cc:330] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:45.605779 26956 raft_consensus.cc:2258] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.605777 26918 raft_consensus.cc:2229] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:45.605857 26957 raft_consensus.cc:2229] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:45.605880 26918 raft_consensus.cc:2258] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.606150 26957 raft_consensus.cc:2258] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.606353 26956 ts_tablet_manager.cc:1875] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.606403 26918 ts_tablet_manager.cc:1875] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.607982 26918 ts_tablet_manager.cc:1888] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:45.607985 26956 ts_tablet_manager.cc:1888] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:45.608048 26956 log.cc:1196] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/wals/92d6b875b1a942dc94cbf54dd42fd2f6
I20240502 07:58:45.608052 26918 log.cc:1196] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/92d6b875b1a942dc94cbf54dd42fd2f6
I20240502 07:58:45.608285 26957 ts_tablet_manager.cc:1875] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.608299 26918 ts_tablet_manager.cc:1909] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:58:45.608373 26956 ts_tablet_manager.cc:1909] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting consensus metadata
I20240502 07:58:45.608906 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet 92d6b875b1a942dc94cbf54dd42fd2f6 (table rebalancer_dynamic_table_new_name [id=2f175365958540dbb77eef3ec4df49bd]) successfully deleted
I20240502 07:58:45.609153 22430 catalog_manager.cc:4830] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): tablet 92d6b875b1a942dc94cbf54dd42fd2f6 (table rebalancer_dynamic_table_new_name [id=2f175365958540dbb77eef3ec4df49bd]) successfully deleted
I20240502 07:58:45.610489 26957 ts_tablet_manager.cc:1888] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:45.610551 26957 log.cc:1196] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/92d6b875b1a942dc94cbf54dd42fd2f6
I20240502 07:58:45.610783 26957 ts_tablet_manager.cc:1909] T 92d6b875b1a942dc94cbf54dd42fd2f6 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:45.611322 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet 92d6b875b1a942dc94cbf54dd42fd2f6 (table rebalancer_dynamic_table_new_name [id=2f175365958540dbb77eef3ec4df49bd]) successfully deleted
I20240502 07:58:45.626123 25265 raft_consensus.cc:1203] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:45.706787 22435 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:45.708182 26079 raft_consensus.cc:1050] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88: attempting to promote NON_VOTER 7eced220cf1642f3873fc385b6191e69 to VOTER
I20240502 07:58:45.708536 26079 consensus_queue.cc:237] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:45.709700 22978 raft_consensus.cc:1261] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:45.710098 25265 raft_consensus.cc:1261] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:45.710505 25532 consensus_queue.cc:1035] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:45.710682 26284 consensus_queue.cc:1035] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:45.712842 24984 tablet_service.cc:1467] Processing CreateTablet for tablet 899eab8bfc0145c2a3b89ac9c24a40ff (DEFAULT_TABLE table=rebalancer_dynamic_table [id=c630f92fc3684f1da7ee98fa7674162c]), partition=
I20240502 07:58:45.712950 22584 raft_consensus.cc:1261] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:45.712962 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 899eab8bfc0145c2a3b89ac9c24a40ff (DEFAULT_TABLE table=rebalancer_dynamic_table [id=c630f92fc3684f1da7ee98fa7674162c]), partition=
I20240502 07:58:45.713140 24984 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 899eab8bfc0145c2a3b89ac9c24a40ff. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.713207 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 899eab8bfc0145c2a3b89ac9c24a40ff. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.713358 26079 consensus_queue.cc:1035] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:45.713690 25532 raft_consensus.cc:2932] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:45.715202 26882 tablet_bootstrap.cc:492] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:45.715329 26879 tablet_bootstrap.cc:492] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:45.715656 22584 raft_consensus.cc:2932] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:45.716163 26879 tablet_bootstrap.cc:654] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:45.716274 26882 tablet_bootstrap.cc:654] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:45.716953 25265 raft_consensus.cc:2932] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:45.717314 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 899eab8bfc0145c2a3b89ac9c24a40ff (DEFAULT_TABLE table=rebalancer_dynamic_table [id=c630f92fc3684f1da7ee98fa7674162c]), partition=
I20240502 07:58:45.717489 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 899eab8bfc0145c2a3b89ac9c24a40ff. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:45.717624 26879 tablet_bootstrap.cc:492] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:45.717691 26879 ts_tablet_manager.cc:1367] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:45.718163 26879 raft_consensus.cc:348] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:45.718255 26879 raft_consensus.cc:374] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:45.718293 26879 raft_consensus.cc:729] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:45.718400 26879 consensus_queue.cc:260] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:45.718844 26879 ts_tablet_manager.cc:1398] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:45.719523 22978 raft_consensus.cc:2932] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:45.720036 22436 catalog_manager.cc:5484] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69 reported cstate change: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:45.720890 26881 tablet_bootstrap.cc:492] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:45.721704 26881 tablet_bootstrap.cc:654] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:45.723591 22847 consensus_queue.cc:237] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:45.724761 22979 raft_consensus.cc:1261] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:45.724988 26881 tablet_bootstrap.cc:492] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:45.725069 26881 ts_tablet_manager.cc:1367] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:45.725486 26881 raft_consensus.cc:348] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:45.725589 26881 raft_consensus.cc:374] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:45.725615 26881 raft_consensus.cc:729] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:45.725720 26881 consensus_queue.cc:260] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:45.725982 26881 ts_tablet_manager.cc:1398] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:45.726250 26079 consensus_queue.cc:1035] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:45.728471 25265 raft_consensus.cc:1261] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:45.728850 26079 consensus_queue.cc:1035] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:45.729166 26284 raft_consensus.cc:2932] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:45.730149 22979 raft_consensus.cc:2932] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:45.731536 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 45499d82e94c47b8b398c159df9fd49b with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:45.732299 25266 raft_consensus.cc:1203] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:45.733595 22440 catalog_manager.cc:5484] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:45.735461 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 45499d82e94c47b8b398c159df9fd49b with delete_type TABLET_DATA_TOMBSTONED (TS 8bea0fe8d8044a2685f92f7a10c9cf85 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:45.735638 25468 tablet_replica.cc:330] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:45.735752 25468 raft_consensus.cc:2229] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:45.735854 25468 raft_consensus.cc:2258] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
W20240502 07:58:45.737134 26961 consensus_queue.cc:1159] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" status { last_received { term: 1 index: 5 } last_committed_idx: 5 last_received_current_leader { term: 1 index: 5 } }
I20240502 07:58:45.737689 25468 ts_tablet_manager.cc:1875] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:45.737628 25265 raft_consensus.cc:2932] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:45.739557 25468 ts_tablet_manager.cc:1888] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:45.739611 25468 log.cc:1196] T 45499d82e94c47b8b398c159df9fd49b P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/45499d82e94c47b8b398c159df9fd49b
I20240502 07:58:45.740089 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 45499d82e94c47b8b398c159df9fd49b (table rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]) successfully deleted
I20240502 07:58:45.741295 26882 tablet_bootstrap.cc:492] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:45.741382 26882 ts_tablet_manager.cc:1367] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.026s user 0.002s sys 0.000s
I20240502 07:58:45.741770 26882 raft_consensus.cc:348] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:45.741859 26882 raft_consensus.cc:374] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:45.741891 26882 raft_consensus.cc:729] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:45.742009 26882 consensus_queue.cc:260] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:45.742229 26882 ts_tablet_manager.cc:1398] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:45.768031 25003 raft_consensus.cc:1203] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:45.788388 25003 raft_consensus.cc:1203] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:45.789098 26696 raft_consensus.cc:482] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:45.789184 26696 raft_consensus.cc:504] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.789582 26696 leader_election.cc:288] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:45.789855 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "325b6a98112d4eef9488d7ef657bc375" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:45.789964 25265 raft_consensus.cc:2454] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:45.790161 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "325b6a98112d4eef9488d7ef657bc375" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:45.790218 24938 leader_election.cc:302] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 74f9aa91c2c448f980b512fe2e8d672e, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:45.790282 25134 raft_consensus.cc:2454] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:45.791244 26696 raft_consensus.cc:2781] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:45.791303 26696 raft_consensus.cc:482] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:45.791342 26696 raft_consensus.cc:3037] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:45.792404 26696 raft_consensus.cc:504] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.792802 26696 leader_election.cc:288] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:45.793385 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "325b6a98112d4eef9488d7ef657bc375" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:45.793500 25265 raft_consensus.cc:3037] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:45.794261 25265 raft_consensus.cc:2454] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:45.794834 24938 leader_election.cc:302] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 74f9aa91c2c448f980b512fe2e8d672e, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:45.794966 26696 raft_consensus.cc:2781] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:45.795053 26696 raft_consensus.cc:686] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Becoming Leader. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Running, Role: LEADER
I20240502 07:58:45.795115 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "325b6a98112d4eef9488d7ef657bc375" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:45.795163 26696 consensus_queue.cc:237] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:45.795203 25134 raft_consensus.cc:3037] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:45.795411 25135 raft_consensus.cc:1203] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:45.796154 25134 raft_consensus.cc:2454] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:45.797994 26696 raft_consensus.cc:482] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:45.798070 26696 raft_consensus.cc:504] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:45.798362 26696 leader_election.cc:288] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:45.798636 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "899eab8bfc0145c2a3b89ac9c24a40ff" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:45.798771 24871 raft_consensus.cc:2454] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:45.798684 22444 catalog_manager.cc:5484] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: term changed from 0 to 1, leader changed from <none> to 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134). New cstate: current_term: 1 leader_uuid: "74f9aa91c2c448f980b512fe2e8d672e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:45.798967 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "899eab8bfc0145c2a3b89ac9c24a40ff" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:45.799069 25265 raft_consensus.cc:2454] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:45.799764 24938 leader_election.cc:302] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 74f9aa91c2c448f980b512fe2e8d672e, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:45.800057 26696 raft_consensus.cc:2781] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:45.800125 26696 raft_consensus.cc:482] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:45.800172 26696 raft_consensus.cc:3037] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:45.800987 26696 raft_consensus.cc:504] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:45.801625 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "899eab8bfc0145c2a3b89ac9c24a40ff" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:45.801743 24871 raft_consensus.cc:3037] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:45.801797 26696 leader_election.cc:288] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:45.801833 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "899eab8bfc0145c2a3b89ac9c24a40ff" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:45.801919 25265 raft_consensus.cc:3037] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:45.802522 24871 raft_consensus.cc:2454] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:45.802592 25265 raft_consensus.cc:2454] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:45.802824 24936 leader_election.cc:302] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:45.802965 26696 raft_consensus.cc:2781] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:45.803050 26696 raft_consensus.cc:686] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Becoming Leader. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Running, Role: LEADER
I20240502 07:58:45.803866 26696 consensus_queue.cc:237] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:45.805624 22444 catalog_manager.cc:5484] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: term changed from 0 to 1, leader changed from <none> to 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134). New cstate: current_term: 1 leader_uuid: "74f9aa91c2c448f980b512fe2e8d672e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:45.808084 25670 raft_consensus.cc:979] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f: : Instructing follower 725969d490a04fdfb3ba890c3f5096e3 to start an election
I20240502 07:58:45.808202 25670 raft_consensus.cc:1067] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Signalling peer 725969d490a04fdfb3ba890c3f5096e3 to start an election
I20240502 07:58:45.808734 22716 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "9d2283e5d28347519b38e876d840608e"
dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
from {username='slave'} at 127.16.85.132:44073
I20240502 07:58:45.808888 22716 raft_consensus.cc:482] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 07:58:45.808955 22716 raft_consensus.cc:3037] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:45.809645 22716 raft_consensus.cc:504] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:45.810036 22716 leader_election.cc:288] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 2 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:45.810298 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d2283e5d28347519b38e876d840608e" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:45.810403 22979 raft_consensus.cc:3032] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Stepping down as leader of term 1
I20240502 07:58:45.810449 22979 raft_consensus.cc:729] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:45.810456 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d2283e5d28347519b38e876d840608e" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:45.810559 24871 raft_consensus.cc:3037] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:45.810581 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d2283e5d28347519b38e876d840608e" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:45.810612 22979 consensus_queue.cc:260] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:45.810679 22847 raft_consensus.cc:3037] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:45.810830 22979 raft_consensus.cc:3037] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:45.811460 24871 raft_consensus.cc:2454] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 2.
I20240502 07:58:45.811654 22847 raft_consensus.cc:2454] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 2.
I20240502 07:58:45.811776 22979 raft_consensus.cc:2454] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 2.
I20240502 07:58:45.811928 22653 leader_election.cc:302] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:45.812125 26995 raft_consensus.cc:2781] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:45.812197 26995 raft_consensus.cc:686] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 2 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:45.812338 26995 consensus_queue.cc:237] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:45.814559 22440 catalog_manager.cc:5484] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 1 to 2, leader changed from 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 2 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:45.822938 24871 raft_consensus.cc:1203] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:45.848938 22847 consensus_queue.cc:237] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
W20240502 07:58:45.850395 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6dbf36c0f044466ab13a53bfe0d6cb53. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:45.850394 22715 raft_consensus.cc:1261] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:45.850690 22584 raft_consensus.cc:1261] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:45.850801 25532 consensus_queue.cc:1035] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:45.851027 26079 consensus_queue.cc:1035] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:45.852416 25532 raft_consensus.cc:2932] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:45.852988 22584 raft_consensus.cc:2932] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:45.855613 22444 catalog_manager.cc:5484] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:45.855825 22715 raft_consensus.cc:2932] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:45.908538 25134 raft_consensus.cc:1203] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:45.914940 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:45.918370 25265 raft_consensus.cc:1261] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:45.918418 24871 raft_consensus.cc:1261] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:45.918787 26697 consensus_queue.cc:1035] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:45.918900 26696 consensus_queue.cc:1035] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:45.920930 22440 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet 899eab8bfc0145c2a3b89ac9c24a40ff (table rebalancer_dynamic_table [id=c630f92fc3684f1da7ee98fa7674162c]). Expected version 1 got 0
I20240502 07:58:45.921030 26800 tablet.cc:1722] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:45.921322 26800 tablet.cc:1620] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e: MemRowSet was empty: no flush needed.
I20240502 07:58:45.922333 22436 catalog_manager.cc:5576] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857) does not have the latest schema for tablet 899eab8bfc0145c2a3b89ac9c24a40ff (table rebalancer_dynamic_table [id=c630f92fc3684f1da7ee98fa7674162c]). Expected version 1 got 0
I20240502 07:58:45.923560 27100 tablet.cc:1722] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:45.923842 27100 tablet.cc:1620] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:45.924855 22436 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet 899eab8bfc0145c2a3b89ac9c24a40ff (table rebalancer_dynamic_table [id=c630f92fc3684f1da7ee98fa7674162c]). Expected version 1 got 0
I20240502 07:58:45.925598 22440 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet 899eab8bfc0145c2a3b89ac9c24a40ff (table rebalancer_dynamic_table [id=c630f92fc3684f1da7ee98fa7674162c]). Expected version 1 got 0
I20240502 07:58:45.926223 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=c630f92fc3684f1da7ee98fa7674162c] alter complete (version 1)
I20240502 07:58:45.929184 26802 tablet.cc:1722] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:45.929507 26802 tablet.cc:1620] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:45.929916 22440 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet 899eab8bfc0145c2a3b89ac9c24a40ff (table rebalancer_dynamic_table [id=c630f92fc3684f1da7ee98fa7674162c]). Expected version 1 got 0
I20240502 07:58:45.933390 25670 consensus_queue.cc:1035] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:45.935148 24871 raft_consensus.cc:1203] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:45.936748 25670 consensus_queue.cc:1035] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:45.970046 22444 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_14" } modify_external_catalogs: true
I20240502 07:58:45.970170 22444 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_14" } modify_external_catalogs: true
I20240502 07:58:45.972187 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 1d382df270004c128dc5718be32532ba
I20240502 07:58:45.972424 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 848fc8384054446aa75a05a52488ea57
I20240502 07:58:45.972774 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet c8c7521bb8414364b927eed84be37621
I20240502 07:58:45.972798 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 1d382df270004c128dc5718be32532ba with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:45.972891 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet 1d382df270004c128dc5718be32532ba with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:45.972981 25114 tablet_service.cc:1514] Processing DeleteTablet for tablet 1d382df270004c128dc5718be32532ba with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:45.973019 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet 848fc8384054446aa75a05a52488ea57 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:45.973045 25468 tablet_replica.cc:330] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:45.973157 26919 tablet_replica.cc:330] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:45.973165 26957 tablet_replica.cc:330] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:45.973183 25468 raft_consensus.cc:2229] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:45.973263 26957 raft_consensus.cc:2229] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:45.973275 26919 raft_consensus.cc:2229] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:45.973349 25468 raft_consensus.cc:2258] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.973381 26957 raft_consensus.cc:2258] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.973481 26919 raft_consensus.cc:2258] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.973855 25468 ts_tablet_manager.cc:1875] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.973935 26957 ts_tablet_manager.cc:1875] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.973971 26919 ts_tablet_manager.cc:1875] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.975679 26919 ts_tablet_manager.cc:1888] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:45.975741 26919 log.cc:1196] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/1d382df270004c128dc5718be32532ba
I20240502 07:58:45.975817 26957 ts_tablet_manager.cc:1888] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:45.975817 25468 ts_tablet_manager.cc:1888] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:45.975876 26957 log.cc:1196] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/1d382df270004c128dc5718be32532ba
I20240502 07:58:45.975876 25468 log.cc:1196] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/1d382df270004c128dc5718be32532ba
I20240502 07:58:45.975997 26919 ts_tablet_manager.cc:1909] T 1d382df270004c128dc5718be32532ba P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:45.976112 25468 ts_tablet_manager.cc:1909] T 1d382df270004c128dc5718be32532ba P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:45.976119 26957 ts_tablet_manager.cc:1909] T 1d382df270004c128dc5718be32532ba P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:45.976459 26919 tablet_replica.cc:330] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:45.976570 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet c8c7521bb8414364b927eed84be37621 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:45.976588 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet 848fc8384054446aa75a05a52488ea57 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:45.976801 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet 848fc8384054446aa75a05a52488ea57 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:45.977183 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet 1d382df270004c128dc5718be32532ba (table rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]) successfully deleted
I20240502 07:58:45.977262 25468 tablet_replica.cc:330] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:45.977373 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 1d382df270004c128dc5718be32532ba (table rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]) successfully deleted
I20240502 07:58:45.977437 24984 tablet_service.cc:1514] Processing DeleteTablet for tablet c8c7521bb8414364b927eed84be37621 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:42878
I20240502 07:58:45.977604 26920 tablet_replica.cc:330] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:45.977607 26956 tablet_replica.cc:330] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e: stopping tablet replica
I20240502 07:58:45.977720 26920 raft_consensus.cc:2229] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:45.977725 25468 raft_consensus.cc:2229] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:45.977725 26956 raft_consensus.cc:2229] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:45.977864 26956 raft_consensus.cc:2258] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.977911 25468 raft_consensus.cc:2258] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.977931 26920 raft_consensus.cc:2258] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.978305 25468 ts_tablet_manager.cc:1875] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.978382 26956 ts_tablet_manager.cc:1875] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.978421 26920 ts_tablet_manager.cc:1875] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.980018 26920 ts_tablet_manager.cc:1888] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:45.980063 26920 log.cc:1196] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/848fc8384054446aa75a05a52488ea57
I20240502 07:58:45.980073 25468 ts_tablet_manager.cc:1888] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:45.980137 25468 log.cc:1196] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/c8c7521bb8414364b927eed84be37621
I20240502 07:58:45.980216 26956 ts_tablet_manager.cc:1888] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:45.980245 26920 ts_tablet_manager.cc:1909] T 848fc8384054446aa75a05a52488ea57 P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:45.980270 26956 log.cc:1196] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/wals/c8c7521bb8414364b927eed84be37621
I20240502 07:58:45.980374 25468 ts_tablet_manager.cc:1909] T c8c7521bb8414364b927eed84be37621 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:45.980432 26918 tablet_replica.cc:330] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:45.980530 26956 ts_tablet_manager.cc:1909] T c8c7521bb8414364b927eed84be37621 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting consensus metadata
I20240502 07:58:45.980563 26918 raft_consensus.cc:2229] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:45.980706 26918 raft_consensus.cc:2258] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.980999 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet c8c7521bb8414364b927eed84be37621 (table rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]) successfully deleted
I20240502 07:58:45.981163 22430 catalog_manager.cc:4830] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): tablet c8c7521bb8414364b927eed84be37621 (table rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]) successfully deleted
I20240502 07:58:45.981220 26918 ts_tablet_manager.cc:1875] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.982657 26918 ts_tablet_manager.cc:1888] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:45.982702 26918 log.cc:1196] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/848fc8384054446aa75a05a52488ea57
I20240502 07:58:45.982878 26918 ts_tablet_manager.cc:1909] T 848fc8384054446aa75a05a52488ea57 P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:58:45.983574 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet c8c7521bb8414364b927eed84be37621 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:45 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:45.983613 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet 848fc8384054446aa75a05a52488ea57 (table rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]) successfully deleted
I20240502 07:58:45.983633 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 1d382df270004c128dc5718be32532ba (table rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]) successfully deleted
I20240502 07:58:45.983863 27126 tablet_replica.cc:330] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:45.983996 27126 raft_consensus.cc:2229] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:45.984143 26919 raft_consensus.cc:2229] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:45.984175 27126 raft_consensus.cc:2258] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.984184 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 848fc8384054446aa75a05a52488ea57 (table rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]) successfully deleted
I20240502 07:58:45.984319 26919 raft_consensus.cc:2258] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:45.984678 27126 ts_tablet_manager.cc:1875] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.984763 26919 ts_tablet_manager.cc:1875] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:45.986225 26919 ts_tablet_manager.cc:1888] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:45.986284 26919 log.cc:1196] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/848fc8384054446aa75a05a52488ea57
I20240502 07:58:45.986542 27126 ts_tablet_manager.cc:1888] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:45.986572 26919 ts_tablet_manager.cc:1909] T 848fc8384054446aa75a05a52488ea57 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:45.986610 27126 log.cc:1196] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/c8c7521bb8414364b927eed84be37621
I20240502 07:58:45.986824 27126 ts_tablet_manager.cc:1909] T c8c7521bb8414364b927eed84be37621 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:45.987066 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 848fc8384054446aa75a05a52488ea57 (table rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]) successfully deleted
I20240502 07:58:45.987367 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet c8c7521bb8414364b927eed84be37621 (table rebalance_test_table_14 [id=08ce30c8d43340468a1d157fb056d3bb]) successfully deleted
I20240502 07:58:45.991618 26696 consensus_queue.cc:1035] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:45.994102 26696 consensus_queue.cc:1035] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:46.030591 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:46.035758 26800 tablet.cc:1722] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:46.036080 26800 tablet.cc:1620] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e: MemRowSet was empty: no flush needed.
I20240502 07:58:46.036195 26802 tablet.cc:1722] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:46.036504 26802 tablet.cc:1620] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:46.036556 27100 tablet.cc:1722] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:46.036834 27100 tablet.cc:1620] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:46.040761 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=c630f92fc3684f1da7ee98fa7674162c] alter complete (version 2)
I20240502 07:58:46.041672 25134 raft_consensus.cc:1203] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:46.050808 25003 raft_consensus.cc:1203] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:46.057530 25265 raft_consensus.cc:1203] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:46.057971 25670 raft_consensus.cc:1050] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 7eced220cf1642f3873fc385b6191e69 to VOTER
I20240502 07:58:46.058351 25670 consensus_queue.cc:237] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:46.059625 22847 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.059633 25265 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.059717 22584 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.059969 25669 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.060111 26128 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.060241 25670 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.061870 26128 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.062183 22847 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.063274 22584 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.064360 22440 catalog_manager.cc:5484] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:46.064896 25265 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.067973 22979 consensus_queue.cc:237] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:46.068884 25265 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.068892 22847 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.069214 25670 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.069353 25669 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.070648 26128 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.071027 22847 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.071856 22429 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ca85b40df34d4e1c91a55d7dd69e1494 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:46.072216 25265 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.072841 22435 catalog_manager.cc:5484] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:46.074808 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet ca85b40df34d4e1c91a55d7dd69e1494 with delete_type TABLET_DATA_TOMBSTONED (TS 8bea0fe8d8044a2685f92f7a10c9cf85 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:46.074980 25468 tablet_replica.cc:330] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:46.075102 25468 raft_consensus.cc:2229] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.075254 25468 raft_consensus.cc:2258] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.075873 25468 ts_tablet_manager.cc:1875] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:46.077581 25468 ts_tablet_manager.cc:1888] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:46.077658 25468 log.cc:1196] T ca85b40df34d4e1c91a55d7dd69e1494 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/ca85b40df34d4e1c91a55d7dd69e1494
I20240502 07:58:46.078143 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet ca85b40df34d4e1c91a55d7dd69e1494 (table rebalance_test_table_3 [id=8f253ed078dd4e948cfdd6ff6c19f874]) successfully deleted
I20240502 07:58:46.090030 22845 raft_consensus.cc:1203] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 07:58:46.141885 27023 raft_consensus.cc:1050] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85: attempting to promote NON_VOTER 7eced220cf1642f3873fc385b6191e69 to VOTER
I20240502 07:58:46.142273 27023 consensus_queue.cc:237] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:46.142307 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:46.143702 22845 raft_consensus.cc:1261] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.143736 22715 raft_consensus.cc:1261] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.143736 25265 raft_consensus.cc:1261] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.144127 25987 consensus_queue.cc:1035] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.144323 27023 consensus_queue.cc:1035] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.144512 24714 consensus_queue.cc:1035] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.147298 27023 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.148044 26800 tablet.cc:1722] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:46.148268 27100 tablet.cc:1722] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:46.148175 22715 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.150504 25266 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.151090 22444 catalog_manager.cc:5484] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:46.153074 26802 tablet.cc:1722] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:46.153676 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=c630f92fc3684f1da7ee98fa7674162c] alter complete (version 3)
I20240502 07:58:46.155306 22845 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.156013 22584 consensus_queue.cc:237] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:46.157621 25266 raft_consensus.cc:1261] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.158041 27136 consensus_queue.cc:1035] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.158213 22716 raft_consensus.cc:1261] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.158561 27136 consensus_queue.cc:1035] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.159722 25987 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.161231 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c53e8b3bd774466dbafe4994472975bc with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:46.161285 22716 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.161334 25266 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.163240 22442 catalog_manager.cc:5484] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:46.165552 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet c53e8b3bd774466dbafe4994472975bc with delete_type TABLET_DATA_TOMBSTONED (TS 7db4178eed174225bfb24f5f9445ce88 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:46.165781 26920 tablet_replica.cc:330] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:46.165923 26920 raft_consensus.cc:2229] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.166051 26920 raft_consensus.cc:2258] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.166900 26920 ts_tablet_manager.cc:1875] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:46.168773 26920 ts_tablet_manager.cc:1888] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:46.168859 26920 log.cc:1196] T c53e8b3bd774466dbafe4994472975bc P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/c53e8b3bd774466dbafe4994472975bc
I20240502 07:58:46.169333 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet c53e8b3bd774466dbafe4994472975bc (table rebalance_test_table_13 [id=2882412f9f504bfc93c45c294c9ace87]) successfully deleted
I20240502 07:58:46.206216 22845 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 07:58:46.206647 26995 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.257758 22442 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:46.257896 22442 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:46.260041 22442 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 899eab8bfc0145c2a3b89ac9c24a40ff
I20240502 07:58:46.260543 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet 899eab8bfc0145c2a3b89ac9c24a40ff with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:46.260748 24984 tablet_service.cc:1514] Processing DeleteTablet for tablet 899eab8bfc0145c2a3b89ac9c24a40ff with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:42878
I20240502 07:58:46.260958 26956 tablet_replica.cc:330] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e: stopping tablet replica
I20240502 07:58:46.261061 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet 899eab8bfc0145c2a3b89ac9c24a40ff with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:46.261106 27230 tablet_replica.cc:330] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:46.261096 26956 raft_consensus.cc:2229] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:46.261210 27230 raft_consensus.cc:2229] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.261270 26956 raft_consensus.cc:2258] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.261317 27230 raft_consensus.cc:2258] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.261436 26918 tablet_replica.cc:330] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:46.261565 26918 raft_consensus.cc:2229] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.261680 26918 raft_consensus.cc:2258] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.261919 27230 ts_tablet_manager.cc:1875] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:46.262107 26956 ts_tablet_manager.cc:1875] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:46.262173 26918 ts_tablet_manager.cc:1875] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:46.263689 26956 ts_tablet_manager.cc:1888] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:46.263753 26956 log.cc:1196] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/wals/899eab8bfc0145c2a3b89ac9c24a40ff
I20240502 07:58:46.263975 26956 ts_tablet_manager.cc:1909] T 899eab8bfc0145c2a3b89ac9c24a40ff P 74f9aa91c2c448f980b512fe2e8d672e: Deleting consensus metadata
I20240502 07:58:46.264087 26918 ts_tablet_manager.cc:1888] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:46.264087 27230 ts_tablet_manager.cc:1888] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:46.264163 26918 log.cc:1196] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/899eab8bfc0145c2a3b89ac9c24a40ff
I20240502 07:58:46.264181 27230 log.cc:1196] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/899eab8bfc0145c2a3b89ac9c24a40ff
I20240502 07:58:46.264468 27230 ts_tablet_manager.cc:1909] T 899eab8bfc0145c2a3b89ac9c24a40ff P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:46.264436 26918 ts_tablet_manager.cc:1909] T 899eab8bfc0145c2a3b89ac9c24a40ff P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:58:46.264760 22430 catalog_manager.cc:4830] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): tablet 899eab8bfc0145c2a3b89ac9c24a40ff (table rebalancer_dynamic_table_new_name [id=c630f92fc3684f1da7ee98fa7674162c]) successfully deleted
I20240502 07:58:46.265129 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet 899eab8bfc0145c2a3b89ac9c24a40ff (table rebalancer_dynamic_table_new_name [id=c630f92fc3684f1da7ee98fa7674162c]) successfully deleted
I20240502 07:58:46.265291 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet 899eab8bfc0145c2a3b89ac9c24a40ff (table rebalancer_dynamic_table_new_name [id=c630f92fc3684f1da7ee98fa7674162c]) successfully deleted
I20240502 07:58:46.269338 26079 raft_consensus.cc:1050] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88: attempting to promote NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 to VOTER
I20240502 07:58:46.269701 26079 consensus_queue.cc:237] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:46.270934 25134 raft_consensus.cc:1261] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.271165 22584 raft_consensus.cc:1261] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.271267 22716 raft_consensus.cc:1261] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.271523 26284 consensus_queue.cc:1035] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.271673 26036 consensus_queue.cc:1035] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.271775 26079 consensus_queue.cc:1035] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.273968 26284 raft_consensus.cc:2932] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.274446 22584 raft_consensus.cc:2932] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.275449 25134 raft_consensus.cc:2932] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.275990 22716 raft_consensus.cc:2932] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.277150 22442 catalog_manager.cc:5484] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:46.280521 22845 consensus_queue.cc:237] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:46.281479 25134 raft_consensus.cc:1261] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.281697 22584 raft_consensus.cc:1261] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.281976 26036 consensus_queue.cc:1035] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.282301 26284 consensus_queue.cc:1035] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.283563 26079 raft_consensus.cc:2932] T 015ed33d8c634473997eaa60e91269a2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.284027 25134 raft_consensus.cc:2932] T 015ed33d8c634473997eaa60e91269a2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.284147 22584 raft_consensus.cc:2932] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.284989 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 015ed33d8c634473997eaa60e91269a2 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:46.286504 22441 catalog_manager.cc:5484] T 015ed33d8c634473997eaa60e91269a2 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.288615 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet 015ed33d8c634473997eaa60e91269a2 with delete_type TABLET_DATA_TOMBSTONED (TS 725969d490a04fdfb3ba890c3f5096e3 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:46.288918 27126 tablet_replica.cc:330] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:46.289053 27126 raft_consensus.cc:2229] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.289209 27126 raft_consensus.cc:2258] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.289937 27126 ts_tablet_manager.cc:1875] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:46.291923 27126 ts_tablet_manager.cc:1888] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:46.292059 27126 log.cc:1196] T 015ed33d8c634473997eaa60e91269a2 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/015ed33d8c634473997eaa60e91269a2
I20240502 07:58:46.292649 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 015ed33d8c634473997eaa60e91269a2 (table rebalance_test_table_1 [id=ed9ff98b498847f8a18550017ae8fc8c]) successfully deleted
I20240502 07:58:46.311662 22435 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_10"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:46.317394 22566 tablet_service.cc:1467] Processing CreateTablet for tablet ed2b3bfe89b24d38b2481bb150b3106a (DEFAULT_TABLE table=rebalancer_extra_table_10 [id=a3c8266f3e52403ca929be65617dd5f3]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:46.317561 25246 tablet_service.cc:1467] Processing CreateTablet for tablet ed2b3bfe89b24d38b2481bb150b3106a (DEFAULT_TABLE table=rebalancer_extra_table_10 [id=a3c8266f3e52403ca929be65617dd5f3]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:46.317747 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ed2b3bfe89b24d38b2481bb150b3106a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:46.317867 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ed2b3bfe89b24d38b2481bb150b3106a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:46.319669 27275 tablet_bootstrap.cc:492] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:46.320040 22959 tablet_service.cc:1467] Processing CreateTablet for tablet da39029564d042f6acfc23f99fb63dfd (DEFAULT_TABLE table=rebalancer_extra_table_10 [id=a3c8266f3e52403ca929be65617dd5f3]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:46.320266 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet da39029564d042f6acfc23f99fb63dfd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:46.320240 22697 tablet_service.cc:1467] Processing CreateTablet for tablet da39029564d042f6acfc23f99fb63dfd (DEFAULT_TABLE table=rebalancer_extra_table_10 [id=a3c8266f3e52403ca929be65617dd5f3]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:46.320240 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 3bb108b6ab55461fb9fabf431d953ef2 (DEFAULT_TABLE table=rebalancer_extra_table_10 [id=a3c8266f3e52403ca929be65617dd5f3]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:46.320457 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet da39029564d042f6acfc23f99fb63dfd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:46.320457 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3bb108b6ab55461fb9fabf431d953ef2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:46.320631 27275 tablet_bootstrap.cc:654] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:46.322088 27278 tablet_bootstrap.cc:492] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:46.322705 25114 tablet_service.cc:1467] Processing CreateTablet for tablet ed2b3bfe89b24d38b2481bb150b3106a (DEFAULT_TABLE table=rebalancer_extra_table_10 [id=a3c8266f3e52403ca929be65617dd5f3]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:46.322795 27275 tablet_bootstrap.cc:492] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:46.322743 25113 tablet_service.cc:1467] Processing CreateTablet for tablet da39029564d042f6acfc23f99fb63dfd (DEFAULT_TABLE table=rebalancer_extra_table_10 [id=a3c8266f3e52403ca929be65617dd5f3]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:46.322860 27275 ts_tablet_manager.cc:1367] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:46.322890 27278 tablet_bootstrap.cc:654] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:46.322960 25114 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ed2b3bfe89b24d38b2481bb150b3106a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:46.323315 27275 raft_consensus.cc:348] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.323416 27275 raft_consensus.cc:374] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:46.323447 27275 raft_consensus.cc:729] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:46.323567 27275 consensus_queue.cc:260] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.323448 27280 tablet_bootstrap.cc:492] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:46.323846 27275 ts_tablet_manager.cc:1398] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:46.323926 27278 tablet_bootstrap.cc:492] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:46.324007 27278 ts_tablet_manager.cc:1367] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:46.324419 27278 raft_consensus.cc:348] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:46.324505 27278 raft_consensus.cc:374] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:46.324530 27278 raft_consensus.cc:729] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:46.324671 27278 consensus_queue.cc:260] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:46.324918 27280 tablet_bootstrap.cc:654] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:46.324971 27278 ts_tablet_manager.cc:1398] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:46.325066 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 3bb108b6ab55461fb9fabf431d953ef2 (DEFAULT_TABLE table=rebalancer_extra_table_10 [id=a3c8266f3e52403ca929be65617dd5f3]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:46.325655 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 3bb108b6ab55461fb9fabf431d953ef2 (DEFAULT_TABLE table=rebalancer_extra_table_10 [id=a3c8266f3e52403ca929be65617dd5f3]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:46.325853 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3bb108b6ab55461fb9fabf431d953ef2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:46.326486 27134 consensus_queue.cc:1035] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:46.327019 24714 raft_consensus.cc:1050] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85: attempting to promote NON_VOTER 6d02017318a2461796d431962fe6a537 to VOTER
I20240502 07:58:46.327353 24714 consensus_queue.cc:237] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:46.327793 27276 tablet_bootstrap.cc:492] T ed2b3bfe89b24d38b2481bb150b3106a P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:46.328596 22979 raft_consensus.cc:1261] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.328634 22845 raft_consensus.cc:1261] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.328763 27276 tablet_bootstrap.cc:654] T ed2b3bfe89b24d38b2481bb150b3106a P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:46.329064 27136 consensus_queue.cc:1035] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.328262 27275 tablet_bootstrap.cc:492] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:46.329595 27136 consensus_queue.cc:1035] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.330144 27275 tablet_bootstrap.cc:654] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:46.330152 27276 tablet_bootstrap.cc:492] T ed2b3bfe89b24d38b2481bb150b3106a P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:46.330230 27276 ts_tablet_manager.cc:1367] T ed2b3bfe89b24d38b2481bb150b3106a P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.001s
I20240502 07:58:46.330616 27276 raft_consensus.cc:348] T ed2b3bfe89b24d38b2481bb150b3106a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.330718 27276 raft_consensus.cc:374] T ed2b3bfe89b24d38b2481bb150b3106a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:46.330811 27276 raft_consensus.cc:729] T ed2b3bfe89b24d38b2481bb150b3106a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:46.330945 27276 consensus_queue.cc:260] T ed2b3bfe89b24d38b2481bb150b3106a P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.331235 27276 ts_tablet_manager.cc:1398] T ed2b3bfe89b24d38b2481bb150b3106a P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:46.332244 27289 tablet_bootstrap.cc:492] T ed2b3bfe89b24d38b2481bb150b3106a P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:46.332722 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3bb108b6ab55461fb9fabf431d953ef2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:46.332752 24871 raft_consensus.cc:1261] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.333307 27289 tablet_bootstrap.cc:654] T ed2b3bfe89b24d38b2481bb150b3106a P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:46.333292 27023 consensus_queue.cc:1035] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.335422 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet da39029564d042f6acfc23f99fb63dfd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:46.335894 25987 raft_consensus.cc:2932] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.336279 24871 raft_consensus.cc:2932] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.338032 27279 tablet_bootstrap.cc:492] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:46.338274 22979 raft_consensus.cc:2932] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.339212 26696 consensus_queue.cc:1035] T 325b6a98112d4eef9488d7ef657bc375 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:46.339509 27279 tablet_bootstrap.cc:654] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:46.339921 27280 tablet_bootstrap.cc:492] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:46.340013 27280 ts_tablet_manager.cc:1367] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.017s user 0.001s sys 0.000s
I20240502 07:58:46.340386 27280 raft_consensus.cc:348] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:46.340480 27280 raft_consensus.cc:374] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:46.340523 27280 raft_consensus.cc:729] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:46.340648 27280 consensus_queue.cc:260] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:46.340953 27280 ts_tablet_manager.cc:1398] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:46.341014 27275 tablet_bootstrap.cc:492] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:46.341089 27275 ts_tablet_manager.cc:1367] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.013s user 0.001s sys 0.000s
I20240502 07:58:46.341506 27275 raft_consensus.cc:348] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.341600 27275 raft_consensus.cc:374] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:46.341629 27275 raft_consensus.cc:729] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:46.341738 27275 consensus_queue.cc:260] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.341986 27275 ts_tablet_manager.cc:1398] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:46.342597 27279 tablet_bootstrap.cc:492] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:46.342674 27279 ts_tablet_manager.cc:1367] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:46.343266 27289 tablet_bootstrap.cc:492] T ed2b3bfe89b24d38b2481bb150b3106a P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:46.343247 27279 raft_consensus.cc:348] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.343375 27279 raft_consensus.cc:374] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:46.343423 27279 raft_consensus.cc:729] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:46.343569 27279 consensus_queue.cc:260] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.343886 27279 ts_tablet_manager.cc:1398] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:46.343861 22440 catalog_manager.cc:5484] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.345078 22845 raft_consensus.cc:2932] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.347594 27289 ts_tablet_manager.cc:1367] T ed2b3bfe89b24d38b2481bb150b3106a P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.015s user 0.002s sys 0.000s
I20240502 07:58:46.348021 27289 raft_consensus.cc:348] T ed2b3bfe89b24d38b2481bb150b3106a P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.348179 27289 raft_consensus.cc:374] T ed2b3bfe89b24d38b2481bb150b3106a P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:46.348260 27289 raft_consensus.cc:729] T ed2b3bfe89b24d38b2481bb150b3106a P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:46.348436 27289 consensus_queue.cc:260] T ed2b3bfe89b24d38b2481bb150b3106a P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.348848 27289 ts_tablet_manager.cc:1398] T ed2b3bfe89b24d38b2481bb150b3106a P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:46.349035 27289 tablet_bootstrap.cc:492] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:46.350100 27289 tablet_bootstrap.cc:654] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:46.350850 22584 consensus_queue.cc:237] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:46.351294 27289 tablet_bootstrap.cc:492] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:46.351374 27289 ts_tablet_manager.cc:1367] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:46.352035 24871 raft_consensus.cc:1261] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.352037 22979 raft_consensus.cc:1261] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.352203 27289 raft_consensus.cc:348] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.352326 27023 consensus_queue.cc:1035] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.352470 25987 consensus_queue.cc:1035] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.352550 27289 raft_consensus.cc:374] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:46.352596 27289 raft_consensus.cc:729] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:46.352880 27289 consensus_queue.cc:260] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.353729 27289 ts_tablet_manager.cc:1398] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:46.353878 27289 tablet_bootstrap.cc:492] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:46.354063 27023 raft_consensus.cc:2932] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.354730 27289 tablet_bootstrap.cc:654] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:46.355034 27136 raft_consensus.cc:1050] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85: attempting to promote NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e to VOTER
I20240502 07:58:46.355005 22979 raft_consensus.cc:2932] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.355360 27136 consensus_queue.cc:237] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:46.355726 27289 tablet_bootstrap.cc:492] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:46.355800 27289 ts_tablet_manager.cc:1367] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.001s
I20240502 07:58:46.356175 27289 raft_consensus.cc:348] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:46.356259 27289 raft_consensus.cc:374] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:46.356288 27289 raft_consensus.cc:729] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:46.356397 27289 consensus_queue.cc:260] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:46.356678 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 887c4c76980d4672aec02b2dbd4d8c89 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:46.356813 27289 ts_tablet_manager.cc:1398] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:46.356959 22716 raft_consensus.cc:1261] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.357491 22979 raft_consensus.cc:1261] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.357522 25003 raft_consensus.cc:1261] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.357831 24714 consensus_queue.cc:1035] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.357923 22435 catalog_manager.cc:5484] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:46.358286 24714 consensus_queue.cc:1035] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.359602 25987 consensus_queue.cc:1035] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.361905 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet 887c4c76980d4672aec02b2dbd4d8c89 with delete_type TABLET_DATA_TOMBSTONED (TS 7db4178eed174225bfb24f5f9445ce88 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:46.362356 26920 tablet_replica.cc:330] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:46.362653 26920 raft_consensus.cc:2229] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.363010 26920 raft_consensus.cc:2258] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.363895 24872 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 2 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 07:58:46.364291 26995 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.364742 26920 ts_tablet_manager.cc:1875] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:46.366320 24871 raft_consensus.cc:2932] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.366736 26920 ts_tablet_manager.cc:1888] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:46.366811 26920 log.cc:1196] T 887c4c76980d4672aec02b2dbd4d8c89 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/887c4c76980d4672aec02b2dbd4d8c89
I20240502 07:58:46.367395 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 887c4c76980d4672aec02b2dbd4d8c89 (table rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]) successfully deleted
I20240502 07:58:46.368522 22978 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 2 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 07:58:46.368914 26389 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.371017 22715 consensus_queue.cc:237] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:46.372354 24871 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 2 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240502 07:58:46.372354 22845 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240502 07:58:46.372714 26389 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:46.373039 26389 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:46.373414 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:46.375279 26995 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.377578 22847 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.378950 22428 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9d2283e5d28347519b38e876d840608e with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:46.379437 22444 catalog_manager.cc:5484] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 5 to 7, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New cstate: current_term: 2 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:46.380436 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 482102b9b3214892a593eb48d1cae943 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=aa5a896c477a4ab48ad2be2757181461]), partition=
I20240502 07:58:46.380690 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 482102b9b3214892a593eb48d1cae943. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:46.380782 24984 tablet_service.cc:1467] Processing CreateTablet for tablet 482102b9b3214892a593eb48d1cae943 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=aa5a896c477a4ab48ad2be2757181461]), partition=
I20240502 07:58:46.381008 24984 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 482102b9b3214892a593eb48d1cae943. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:46.382498 27289 tablet_bootstrap.cc:492] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:46.383319 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 482102b9b3214892a593eb48d1cae943 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=aa5a896c477a4ab48ad2be2757181461]), partition=
I20240502 07:58:46.383379 27289 tablet_bootstrap.cc:654] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:46.383528 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 482102b9b3214892a593eb48d1cae943. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:46.383724 24871 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.384552 27305 tablet_bootstrap.cc:492] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:46.385355 27305 tablet_bootstrap.cc:654] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:46.386559 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet 9d2283e5d28347519b38e876d840608e with delete_type TABLET_DATA_TOMBSTONED (TS 02a92e1c084b4308b9e855a4cc2c295f not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:46.386973 26919 tablet_replica.cc:330] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:46.387128 26919 raft_consensus.cc:2229] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.387246 26919 raft_consensus.cc:2258] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.387220 27023 raft_consensus.cc:2932] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.387840 26919 ts_tablet_manager.cc:1875] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:46.389171 25003 raft_consensus.cc:2932] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.389416 22979 raft_consensus.cc:2932] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.389717 26919 ts_tablet_manager.cc:1888] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 07:58:46.389776 26919 log.cc:1196] T 9d2283e5d28347519b38e876d840608e P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/9d2283e5d28347519b38e876d840608e
I20240502 07:58:46.390758 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 9d2283e5d28347519b38e876d840608e (table rebalance_test_table_11 [id=0c1156cac15d48028ebf5c07c67ea832]) successfully deleted
I20240502 07:58:46.390903 27279 tablet_bootstrap.cc:492] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:46.391757 27279 tablet_bootstrap.cc:654] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:46.392145 22441 catalog_manager.cc:5484] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.393260 27279 tablet_bootstrap.cc:492] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:46.393338 27279 ts_tablet_manager.cc:1367] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:46.393798 22716 raft_consensus.cc:2932] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.394060 25669 raft_consensus.cc:1050] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 6d02017318a2461796d431962fe6a537 to VOTER
I20240502 07:58:46.394347 25669 consensus_queue.cc:237] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:46.395658 24871 raft_consensus.cc:1261] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.395834 22715 raft_consensus.cc:1261] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.396209 25670 consensus_queue.cc:1035] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.396452 22847 raft_consensus.cc:1261] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.396642 26128 consensus_queue.cc:1035] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.396853 25670 consensus_queue.cc:1035] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.399158 22584 consensus_queue.cc:237] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:46.400504 27289 tablet_bootstrap.cc:492] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:46.400602 27289 ts_tablet_manager.cc:1367] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.018s user 0.002s sys 0.000s
I20240502 07:58:46.400844 25003 raft_consensus.cc:1261] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.401012 27289 raft_consensus.cc:348] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:46.401115 27289 raft_consensus.cc:374] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:46.401139 27289 raft_consensus.cc:729] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:46.401228 27289 consensus_queue.cc:260] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:46.401269 25987 consensus_queue.cc:1035] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.401458 27289 ts_tablet_manager.cc:1398] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:46.402895 27002 raft_consensus.cc:482] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:46.402994 27002 raft_consensus.cc:504] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.403280 22716 raft_consensus.cc:1261] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.403491 27002 leader_election.cc:288] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:46.403640 25987 consensus_queue.cc:1035] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.403831 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ed2b3bfe89b24d38b2481bb150b3106a" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:46.403928 25134 raft_consensus.cc:2454] T ed2b3bfe89b24d38b2481bb150b3106a P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:46.404299 25202 leader_election.cc:302] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:46.404495 27002 raft_consensus.cc:2781] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:46.404539 27002 raft_consensus.cc:482] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:46.404572 27002 raft_consensus.cc:3037] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:46.405381 27301 raft_consensus.cc:1050] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3: attempting to promote NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e to VOTER
I20240502 07:58:46.405530 27002 raft_consensus.cc:504] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.405493 25987 raft_consensus.cc:2932] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.405640 27279 raft_consensus.cc:348] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:46.405776 27279 raft_consensus.cc:374] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:46.405814 27279 raft_consensus.cc:729] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:46.405777 27301 consensus_queue.cc:237] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:46.405850 27002 leader_election.cc:288] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:46.405942 27279 consensus_queue.cc:260] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:46.406397 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ed2b3bfe89b24d38b2481bb150b3106a" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:46.406508 25134 raft_consensus.cc:3037] T ed2b3bfe89b24d38b2481bb150b3106a P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:46.407163 22847 raft_consensus.cc:1261] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.407176 22979 raft_consensus.cc:1261] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.407390 25134 raft_consensus.cc:2454] T ed2b3bfe89b24d38b2481bb150b3106a P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:46.407616 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b583c8775a874ad1a89c82c00c05bbbd with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:46.407588 27301 consensus_queue.cc:1035] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.407698 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ed2b3bfe89b24d38b2481bb150b3106a" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:46.407752 26389 consensus_queue.cc:1035] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.407785 22586 raft_consensus.cc:3037] T ed2b3bfe89b24d38b2481bb150b3106a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:46.408022 25003 raft_consensus.cc:1261] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.408284 22716 raft_consensus.cc:2932] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.408705 22586 raft_consensus.cc:2454] T ed2b3bfe89b24d38b2481bb150b3106a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:46.409133 25202 leader_election.cc:302] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7eced220cf1642f3873fc385b6191e69, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:46.409483 27301 consensus_queue.cc:1035] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.409960 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ed2b3bfe89b24d38b2481bb150b3106a" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:46.410080 22584 raft_consensus.cc:2362] T ed2b3bfe89b24d38b2481bb150b3106a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1. Re-sending same reply.
I20240502 07:58:46.411231 25004 raft_consensus.cc:2932] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.411370 27279 ts_tablet_manager.cc:1398] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.018s user 0.001s sys 0.000s
I20240502 07:58:46.411623 22440 catalog_manager.cc:5484] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.412834 26995 raft_consensus.cc:2932] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.413254 25003 raft_consensus.cc:2932] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.414469 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet b583c8775a874ad1a89c82c00c05bbbd with delete_type TABLET_DATA_TOMBSTONED (TS 02a92e1c084b4308b9e855a4cc2c295f not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:46.414717 22847 raft_consensus.cc:2932] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.416401 26919 tablet_replica.cc:330] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:46.416555 26919 raft_consensus.cc:2229] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.416726 26919 raft_consensus.cc:2258] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.417304 26919 ts_tablet_manager.cc:1875] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:46.417276 22440 catalog_manager.cc:5484] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.418823 27002 raft_consensus.cc:2781] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:46.419021 27002 raft_consensus.cc:686] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:46.419173 27002 consensus_queue.cc:237] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.419245 26919 ts_tablet_manager.cc:1888] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:46.419307 26919 log.cc:1196] T b583c8775a874ad1a89c82c00c05bbbd P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/b583c8775a874ad1a89c82c00c05bbbd
I20240502 07:58:46.419816 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet b583c8775a874ad1a89c82c00c05bbbd (table rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]) successfully deleted
I20240502 07:58:46.420939 26128 raft_consensus.cc:1050] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 to VOTER
I20240502 07:58:46.421247 26128 consensus_queue.cc:237] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:46.421871 27305 tablet_bootstrap.cc:492] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:46.421952 27305 ts_tablet_manager.cc:1367] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.037s user 0.001s sys 0.000s
I20240502 07:58:46.422370 27305 raft_consensus.cc:348] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:46.422446 27305 raft_consensus.cc:374] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:46.422467 27305 raft_consensus.cc:729] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:46.422379 22441 catalog_manager.cc:5484] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:46.422560 27305 consensus_queue.cc:260] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:46.422646 22715 consensus_queue.cc:237] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:46.422808 27305 ts_tablet_manager.cc:1398] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:46.423240 22847 raft_consensus.cc:1261] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.423624 25669 consensus_queue.cc:1035] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.423833 22715 raft_consensus.cc:1261] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.424168 25670 consensus_queue.cc:1035] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.424239 25003 raft_consensus.cc:1261] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.424465 25134 raft_consensus.cc:1261] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.424599 27301 consensus_queue.cc:1035] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.425273 25670 consensus_queue.cc:1035] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.425746 22847 raft_consensus.cc:1261] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.426069 27301 consensus_queue.cc:1035] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.427834 27301 raft_consensus.cc:2932] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.427906 22715 raft_consensus.cc:2932] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.429054 25003 raft_consensus.cc:2932] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.430895 22441 catalog_manager.cc:5484] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.431246 25134 raft_consensus.cc:2932] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.433048 25669 raft_consensus.cc:1050] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e to VOTER
I20240502 07:58:46.432966 22440 catalog_manager.cc:5484] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.433364 25669 consensus_queue.cc:237] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:46.434650 25003 raft_consensus.cc:1261] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.434820 22428 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 539cf5d4596f481cb03ab4d13a19198c with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:46.435072 25669 consensus_queue.cc:1035] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.435710 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet 539cf5d4596f481cb03ab4d13a19198c with delete_type TABLET_DATA_TOMBSTONED (TS 02a92e1c084b4308b9e855a4cc2c295f not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:46.435981 22584 raft_consensus.cc:1261] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.436416 22715 raft_consensus.cc:1261] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.436635 27325 consensus_queue.cc:1035] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.437067 27325 consensus_queue.cc:1035] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.437033 22847 raft_consensus.cc:2932] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.439980 25669 raft_consensus.cc:2932] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.440524 25003 raft_consensus.cc:2932] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.441535 26919 tablet_replica.cc:330] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:46.441643 26919 raft_consensus.cc:2229] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.441732 26919 pending_rounds.cc:62] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f: Trying to abort 1 pending ops.
I20240502 07:58:46.441644 22584 raft_consensus.cc:2932] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.441784 26919 pending_rounds.cc:68] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f: Aborting op as it isn't in flight: id { term: 1 index: 5 } timestamp: 7023152031356870656 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "539cf5d4596f481cb03ab4d13a19198c" old_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } } new_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } } }
I20240502 07:58:46.441884 26919 raft_consensus.cc:2894] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Aborting config change with OpId 1.5: Aborted: Op aborted
I20240502 07:58:46.441933 26919 raft_consensus.cc:2866] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 07:58:46.441970 26919 raft_consensus.cc:2258] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.427580 27310 raft_consensus.cc:2932] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.442337 27110 raft_consensus.cc:482] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:46.442412 27110 raft_consensus.cc:504] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.442543 26919 ts_tablet_manager.cc:1875] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:46.442849 27110 leader_election.cc:288] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:46.444299 22441 catalog_manager.cc:5484] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.444715 26919 ts_tablet_manager.cc:1888] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:46.444846 26919 log.cc:1196] T 539cf5d4596f481cb03ab4d13a19198c P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/539cf5d4596f481cb03ab4d13a19198c
I20240502 07:58:46.444792 22716 raft_consensus.cc:2932] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.445860 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 539cf5d4596f481cb03ab4d13a19198c (table rebalance_test_table_15 [id=4b1b337aafc04dc69a2e37f348a5652a]) successfully deleted
I20240502 07:58:46.448587 25669 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.450450 22979 consensus_queue.cc:237] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:46.451325 22440 catalog_manager.cc:5484] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:46.451589 25003 raft_consensus.cc:1261] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.451597 22715 raft_consensus.cc:1261] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.452029 27325 consensus_queue.cc:1035] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.452404 26128 consensus_queue.cc:1035] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.452500 22716 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.452975 22845 raft_consensus.cc:2932] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.454592 25670 raft_consensus.cc:2932] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.454957 25003 raft_consensus.cc:2932] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.456959 22429 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet eb0a7b1308114769b556795ae18a746e with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:46.457864 22441 catalog_manager.cc:5484] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.458333 22847 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.458760 22715 raft_consensus.cc:2932] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:46.460032 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet eb0a7b1308114769b556795ae18a746e with delete_type TABLET_DATA_TOMBSTONED (TS 8bea0fe8d8044a2685f92f7a10c9cf85 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:46.460646 25468 tablet_replica.cc:330] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:46.460813 25468 raft_consensus.cc:2229] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.460978 25468 raft_consensus.cc:2258] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.461839 25468 ts_tablet_manager.cc:1875] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:46.461901 22979 consensus_queue.cc:237] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:46.463081 25134 raft_consensus.cc:1261] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.463181 22715 raft_consensus.cc:1261] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.463570 26128 consensus_queue.cc:1035] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.463748 27325 consensus_queue.cc:1035] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.463804 25468 ts_tablet_manager.cc:1888] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:46.463861 25468 log.cc:1196] T eb0a7b1308114769b556795ae18a746e P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/eb0a7b1308114769b556795ae18a746e
I20240502 07:58:46.464391 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet eb0a7b1308114769b556795ae18a746e (table rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]) successfully deleted
I20240502 07:58:46.465548 25670 raft_consensus.cc:2932] T e24e4e639b0649e3ac135ac9358a0398 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.465864 25135 raft_consensus.cc:2932] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.467559 22429 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e24e4e639b0649e3ac135ac9358a0398 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:46.467819 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3bb108b6ab55461fb9fabf431d953ef2" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:46.467947 25266 raft_consensus.cc:2454] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:46.468075 22715 raft_consensus.cc:2932] T e24e4e639b0649e3ac135ac9358a0398 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.468216 27304 ts_tablet_manager.cc:902] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e: Initiating tablet copy from peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:46.468142 22435 catalog_manager.cc:5484] T e24e4e639b0649e3ac135ac9358a0398 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.468218 24807 leader_election.cc:302] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:46.468375 27110 raft_consensus.cc:2781] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:46.468416 27110 raft_consensus.cc:482] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:46.468446 27110 raft_consensus.cc:3037] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:46.468699 27304 tablet_copy_client.cc:308] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.131:44055
I20240502 07:58:46.469080 22868 tablet_copy_service.cc:140] P 7db4178eed174225bfb24f5f9445ce88: Received BeginTabletCopySession request for tablet 6dbf36c0f044466ab13a53bfe0d6cb53 from peer 74f9aa91c2c448f980b512fe2e8d672e ({username='slave'} at 127.16.85.134:53609)
I20240502 07:58:46.469174 22868 tablet_copy_service.cc:161] P 7db4178eed174225bfb24f5f9445ce88: Beginning new tablet copy session on tablet 6dbf36c0f044466ab13a53bfe0d6cb53 from peer 74f9aa91c2c448f980b512fe2e8d672e at {username='slave'} at 127.16.85.134:53609: session id = 74f9aa91c2c448f980b512fe2e8d672e-6dbf36c0f044466ab13a53bfe0d6cb53
I20240502 07:58:46.469372 27110 raft_consensus.cc:504] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.469687 27110 leader_election.cc:288] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:46.470037 22868 tablet_copy_source_session.cc:205] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:46.470201 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3bb108b6ab55461fb9fabf431d953ef2" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:46.470293 25266 raft_consensus.cc:3037] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:46.471047 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet e24e4e639b0649e3ac135ac9358a0398 with delete_type TABLET_DATA_TOMBSTONED (TS 7db4178eed174225bfb24f5f9445ce88 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:46.471153 25266 raft_consensus.cc:2454] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:46.471227 26920 tablet_replica.cc:330] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:46.471346 26920 raft_consensus.cc:2229] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.471390 24807 leader_election.cc:302] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:46.471500 26920 raft_consensus.cc:2258] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.471397 24871 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.472306 26920 ts_tablet_manager.cc:1875] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:46.472944 27304 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6dbf36c0f044466ab13a53bfe0d6cb53. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:46.473040 27339 raft_consensus.cc:2781] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:46.473107 27339 raft_consensus.cc:686] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:58:46.473253 27339 consensus_queue.cc:237] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:46.474231 26920 ts_tablet_manager.cc:1888] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:46.474237 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3bb108b6ab55461fb9fabf431d953ef2" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:46.474310 26920 log.cc:1196] T e24e4e639b0649e3ac135ac9358a0398 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/e24e4e639b0649e3ac135ac9358a0398
I20240502 07:58:46.474364 25134 raft_consensus.cc:3037] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:46.474756 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet e24e4e639b0649e3ac135ac9358a0398 (table rebalance_test_table_3 [id=8f253ed078dd4e948cfdd6ff6c19f874]) successfully deleted
I20240502 07:58:46.475219 25134 raft_consensus.cc:2454] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:46.475149 27304 tablet_copy_client.cc:783] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:46.475495 27304 tablet_copy_client.cc:647] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:46.476075 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3bb108b6ab55461fb9fabf431d953ef2" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:46.476193 25135 raft_consensus.cc:2362] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1. Re-sending same reply.
I20240502 07:58:46.476750 22435 catalog_manager.cc:5484] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:46.477929 27304 tablet_copy_client.cc:523] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:46.479600 22979 consensus_queue.cc:237] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:46.479799 27304 tablet_bootstrap.cc:492] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:46.480945 24871 raft_consensus.cc:1261] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.481252 22847 raft_consensus.cc:1261] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.481251 27333 consensus_queue.cc:1035] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.481597 27310 consensus_queue.cc:1035] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.482592 25670 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.483299 22847 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.483857 22441 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_16" } modify_external_catalogs: true
I20240502 07:58:46.483953 22441 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_16" } modify_external_catalogs: true
I20240502 07:58:46.484117 22429 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e5d5f67e026344998d5fb079f532d076 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:46.485036 22435 catalog_manager.cc:5484] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:46.485775 24871 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:46.487543 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet e5d5f67e026344998d5fb079f532d076 with delete_type TABLET_DATA_TOMBSTONED (TS 725969d490a04fdfb3ba890c3f5096e3 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:46.487701 27126 tablet_replica.cc:330] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:46.487813 27126 raft_consensus.cc:2229] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.487928 27126 raft_consensus.cc:2258] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.488014 22441 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 905422dc16e74174b4f88aa87cbee3d3
I20240502 07:58:46.488222 22441 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet eb0a7b1308114769b556795ae18a746e
I20240502 07:58:46.488497 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet 905422dc16e74174b4f88aa87cbee3d3 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:46.488631 26920 tablet_replica.cc:330] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:46.488590 25113 tablet_service.cc:1514] Processing DeleteTablet for tablet 905422dc16e74174b4f88aa87cbee3d3 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:46.488770 26920 raft_consensus.cc:2229] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.488898 26920 raft_consensus.cc:2258] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.488953 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 905422dc16e74174b4f88aa87cbee3d3 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:46.489066 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet eb0a7b1308114769b556795ae18a746e with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:46.489096 25468 tablet_replica.cc:330] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:46.489194 25468 raft_consensus.cc:2229] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:46.489306 26920 ts_tablet_manager.cc:1875] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:46.489365 25468 raft_consensus.cc:2258] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.489480 27126 ts_tablet_manager.cc:1875] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:46.489820 25468 ts_tablet_manager.cc:1875] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:46.491057 26920 ts_tablet_manager.cc:1888] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:46.491118 26920 log.cc:1196] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/905422dc16e74174b4f88aa87cbee3d3
I20240502 07:58:46.491374 26920 ts_tablet_manager.cc:1909] T 905422dc16e74174b4f88aa87cbee3d3 P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:46.491492 27126 ts_tablet_manager.cc:1888] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:46.491556 27126 log.cc:1196] T e5d5f67e026344998d5fb079f532d076 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/e5d5f67e026344998d5fb079f532d076
I20240502 07:58:46.491735 25468 ts_tablet_manager.cc:1888] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:46.491796 25468 log.cc:1196] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/905422dc16e74174b4f88aa87cbee3d3
I20240502 07:58:46.491940 27126 tablet_replica.cc:330] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:46.492024 25468 ts_tablet_manager.cc:1909] T 905422dc16e74174b4f88aa87cbee3d3 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:46.492048 27126 raft_consensus.cc:2229] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.492187 27126 raft_consensus.cc:2258] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.492283 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet eb0a7b1308114769b556795ae18a746e with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:46.492208 22441 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 887c4c76980d4672aec02b2dbd4d8c89
I20240502 07:58:46.492619 24984 tablet_service.cc:1514] Processing DeleteTablet for tablet eb0a7b1308114769b556795ae18a746e with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:42878
I20240502 07:58:46.492769 26956 tablet_replica.cc:330] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e: stopping tablet replica
I20240502 07:58:46.492841 26919 tablet_replica.cc:330] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:46.492879 26956 raft_consensus.cc:2229] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.492946 26919 raft_consensus.cc:2229] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:46.492987 26956 raft_consensus.cc:2258] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.493098 26919 raft_consensus.cc:2258] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.493387 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet 887c4c76980d4672aec02b2dbd4d8c89 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:46.493544 27230 tablet_replica.cc:330] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:46.493522 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet 887c4c76980d4672aec02b2dbd4d8c89 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:46.493649 27230 raft_consensus.cc:2229] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.493752 27230 raft_consensus.cc:2258] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.493835 26919 ts_tablet_manager.cc:1875] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:46.493940 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 905422dc16e74174b4f88aa87cbee3d3 (table rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]) successfully deleted
I20240502 07:58:46.494621 26956 ts_tablet_manager.cc:1875] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:46.495689 26919 ts_tablet_manager.cc:1888] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:46.495749 26919 log.cc:1196] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/eb0a7b1308114769b556795ae18a746e
I20240502 07:58:46.495965 26919 ts_tablet_manager.cc:1909] T eb0a7b1308114769b556795ae18a746e P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:46.496412 26956 ts_tablet_manager.cc:1888] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:46.496443 26919 tablet_replica.cc:330] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:46.496472 26956 log.cc:1196] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/wals/eb0a7b1308114769b556795ae18a746e
I20240502 07:58:46.496527 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet eb0a7b1308114769b556795ae18a746e (table rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]) successfully deleted
I20240502 07:58:46.496553 26919 raft_consensus.cc:2229] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.496884 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 887c4c76980d4672aec02b2dbd4d8c89 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:46.497170 27230 ts_tablet_manager.cc:1875] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:46.497458 25468 tablet_replica.cc:330] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:46.497547 25468 raft_consensus.cc:2229] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:46.497701 25468 raft_consensus.cc:2258] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.497859 26919 raft_consensus.cc:2258] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.498502 26919 ts_tablet_manager.cc:1875] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:46.499022 27230 ts_tablet_manager.cc:1888] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:46.499075 27230 log.cc:1196] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/887c4c76980d4672aec02b2dbd4d8c89
I20240502 07:58:46.499305 27230 ts_tablet_manager.cc:1909] T 887c4c76980d4672aec02b2dbd4d8c89 P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:46.499926 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet 887c4c76980d4672aec02b2dbd4d8c89 (table rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]) successfully deleted
I20240502 07:58:46.500043 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet e5d5f67e026344998d5fb079f532d076 (table rebalance_test_table_1 [id=ed9ff98b498847f8a18550017ae8fc8c]) successfully deleted
I20240502 07:58:46.500164 26919 ts_tablet_manager.cc:1888] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:46.500217 26919 log.cc:1196] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/887c4c76980d4672aec02b2dbd4d8c89
I20240502 07:58:46.500247 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 905422dc16e74174b4f88aa87cbee3d3 (table rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]) successfully deleted
I20240502 07:58:46.500461 26919 ts_tablet_manager.cc:1909] T 887c4c76980d4672aec02b2dbd4d8c89 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:46.500859 25468 ts_tablet_manager.cc:1875] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:46.501677 26956 ts_tablet_manager.cc:1909] T eb0a7b1308114769b556795ae18a746e P 74f9aa91c2c448f980b512fe2e8d672e: Deleting consensus metadata
I20240502 07:58:46.502233 25468 ts_tablet_manager.cc:1888] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:46.502264 22430 catalog_manager.cc:4830] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): tablet eb0a7b1308114769b556795ae18a746e (table rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]) successfully deleted
I20240502 07:58:46.502300 25468 log.cc:1196] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/887c4c76980d4672aec02b2dbd4d8c89
I20240502 07:58:46.502514 25468 ts_tablet_manager.cc:1909] T 887c4c76980d4672aec02b2dbd4d8c89 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:46.502899 27126 ts_tablet_manager.cc:1875] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:46.503033 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 887c4c76980d4672aec02b2dbd4d8c89 (table rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]) successfully deleted
I20240502 07:58:46.503398 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 887c4c76980d4672aec02b2dbd4d8c89 (table rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]) successfully deleted
I20240502 07:58:46.504809 27126 ts_tablet_manager.cc:1888] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:46.504878 27126 log.cc:1196] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/eb0a7b1308114769b556795ae18a746e
I20240502 07:58:46.505105 27126 ts_tablet_manager.cc:1909] T eb0a7b1308114769b556795ae18a746e P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:46.505693 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet eb0a7b1308114769b556795ae18a746e (table rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]) successfully deleted
I20240502 07:58:46.506166 27343 tablet_replica.cc:330] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:46.506318 27343 raft_consensus.cc:2229] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.506449 27343 raft_consensus.cc:2258] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.506920 27343 ts_tablet_manager.cc:1875] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:46.508039 27304 tablet_bootstrap.cc:492] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:46.508325 27304 tablet_bootstrap.cc:492] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap complete.
I20240502 07:58:46.508533 27304 ts_tablet_manager.cc:1367] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.029s user 0.023s sys 0.000s
I20240502 07:58:46.509002 27304 raft_consensus.cc:348] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:46.509104 27136 raft_consensus.cc:1050] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85: attempting to promote NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 to VOTER
I20240502 07:58:46.509112 27304 raft_consensus.cc:729] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: LEARNER
I20240502 07:58:46.509218 27304 consensus_queue.cc:260] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:46.509307 27343 ts_tablet_manager.cc:1888] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:46.509403 27343 log.cc:1196] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/905422dc16e74174b4f88aa87cbee3d3
I20240502 07:58:46.509440 27136 consensus_queue.cc:237] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:46.509616 27304 ts_tablet_manager.cc:1398] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:46.509687 27343 ts_tablet_manager.cc:1909] T 905422dc16e74174b4f88aa87cbee3d3 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:46.510138 22868 tablet_copy_service.cc:342] P 7db4178eed174225bfb24f5f9445ce88: Request end of tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-6dbf36c0f044466ab13a53bfe0d6cb53 received from {username='slave'} at 127.16.85.134:53609
I20240502 07:58:46.510206 22868 tablet_copy_service.cc:433] P 7db4178eed174225bfb24f5f9445ce88: ending tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-6dbf36c0f044466ab13a53bfe0d6cb53 on tablet 6dbf36c0f044466ab13a53bfe0d6cb53 with peer 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:46.510650 22979 raft_consensus.cc:1261] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.511025 24714 consensus_queue.cc:1035] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.511716 22847 raft_consensus.cc:1261] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.512149 24714 consensus_queue.cc:1035] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.512588 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet 905422dc16e74174b4f88aa87cbee3d3 (table rebalance_test_table_16 [id=566f71aea84b4c13860918ee76472349]) successfully deleted
I20240502 07:58:46.512807 25135 raft_consensus.cc:1261] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.513370 24714 consensus_queue.cc:1035] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.513846 27136 raft_consensus.cc:2932] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.514883 22847 raft_consensus.cc:2932] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.515409 22979 raft_consensus.cc:2932] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.517246 25135 raft_consensus.cc:2932] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.517477 22440 catalog_manager.cc:5484] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.521904 22584 consensus_queue.cc:237] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:46.522935 25135 raft_consensus.cc:1261] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.523010 22979 raft_consensus.cc:1261] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.523355 27023 consensus_queue.cc:1035] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.523543 27136 consensus_queue.cc:1035] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.524964 24714 raft_consensus.cc:2932] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.526494 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9ca2e5b5a0d24bcc9d1b69f998a0d928 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:46.527033 25135 raft_consensus.cc:2932] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.527122 22979 raft_consensus.cc:2932] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:46.527241 22441 catalog_manager.cc:5484] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:46.529536 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet 9ca2e5b5a0d24bcc9d1b69f998a0d928 with delete_type TABLET_DATA_TOMBSTONED (TS 7db4178eed174225bfb24f5f9445ce88 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:46.529718 26920 tablet_replica.cc:330] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:46.529843 26920 raft_consensus.cc:2229] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.529999 26920 raft_consensus.cc:2258] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.530591 26920 ts_tablet_manager.cc:1875] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:46.532775 26920 ts_tablet_manager.cc:1888] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:46.532850 26920 log.cc:1196] T 9ca2e5b5a0d24bcc9d1b69f998a0d928 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/9ca2e5b5a0d24bcc9d1b69f998a0d928
I20240502 07:58:46.533469 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 9ca2e5b5a0d24bcc9d1b69f998a0d928 (table rebalance_test_table_7 [id=b66f182129cf49c0a53671ec66e4a6e2]) successfully deleted
I20240502 07:58:46.566982 26079 raft_consensus.cc:1050] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88: attempting to promote NON_VOTER 7eced220cf1642f3873fc385b6191e69 to VOTER
I20240502 07:58:46.567346 26079 consensus_queue.cc:237] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:46.568554 22979 raft_consensus.cc:1261] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.568647 22715 raft_consensus.cc:1261] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.568791 25266 raft_consensus.cc:1261] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:46.569039 26284 consensus_queue.cc:1035] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.569200 25532 consensus_queue.cc:1035] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.569497 25532 consensus_queue.cc:1035] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:46.573661 26284 raft_consensus.cc:2932] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.574040 22715 raft_consensus.cc:2932] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.576448 22444 catalog_manager.cc:5484] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.578765 22979 raft_consensus.cc:2932] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.581693 25266 raft_consensus.cc:2932] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.587574 22847 consensus_queue.cc:237] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:46.588647 25266 raft_consensus.cc:1261] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.588928 27303 raft_consensus.cc:482] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:46.589010 27303 raft_consensus.cc:504] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:46.589049 22979 raft_consensus.cc:1261] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:46.589395 27303 leader_election.cc:288] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:46.589390 25532 consensus_queue.cc:1035] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.589823 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "da39029564d042f6acfc23f99fb63dfd" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:46.589807 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "da39029564d042f6acfc23f99fb63dfd" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:46.589916 22979 raft_consensus.cc:2454] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:46.589926 25135 raft_consensus.cc:2454] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:46.590142 22651 leader_election.cc:302] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:46.590348 27303 raft_consensus.cc:2781] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:46.590392 27303 raft_consensus.cc:482] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:46.590432 27303 raft_consensus.cc:3037] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:46.590448 25532 consensus_queue.cc:1035] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:46.591354 27303 raft_consensus.cc:504] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:46.591639 27303 leader_election.cc:288] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:46.591926 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "da39029564d042f6acfc23f99fb63dfd" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:46.592028 25135 raft_consensus.cc:3037] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:46.592173 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "da39029564d042f6acfc23f99fb63dfd" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:46.592272 22979 raft_consensus.cc:3037] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:46.593107 22979 raft_consensus.cc:2454] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:46.593256 25135 raft_consensus.cc:2454] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:46.593355 22651 leader_election.cc:302] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:46.593698 27303 raft_consensus.cc:2781] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:46.593786 27303 raft_consensus.cc:686] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:46.593892 27303 consensus_queue.cc:237] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:46.595127 25532 raft_consensus.cc:2932] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.595626 22435 catalog_manager.cc:5484] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:46.596886 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet abfc26b86b17464fa7cf0b13ab73861c with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:46.598361 22444 catalog_manager.cc:5484] T abfc26b86b17464fa7cf0b13ab73861c P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:46.599171 25266 raft_consensus.cc:2932] T abfc26b86b17464fa7cf0b13ab73861c P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.601253 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet abfc26b86b17464fa7cf0b13ab73861c with delete_type TABLET_DATA_TOMBSTONED (TS 725969d490a04fdfb3ba890c3f5096e3 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:46.601609 27126 tablet_replica.cc:330] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:46.601745 27126 raft_consensus.cc:2229] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.601919 27126 raft_consensus.cc:2258] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.602370 22979 raft_consensus.cc:2932] T abfc26b86b17464fa7cf0b13ab73861c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:46.604413 27126 ts_tablet_manager.cc:1875] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:46.606585 27126 ts_tablet_manager.cc:1888] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:46.606652 27126 log.cc:1196] T abfc26b86b17464fa7cf0b13ab73861c P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/abfc26b86b17464fa7cf0b13ab73861c
I20240502 07:58:46.607154 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet abfc26b86b17464fa7cf0b13ab73861c (table rebalance_test_table_7 [id=b66f182129cf49c0a53671ec66e4a6e2]) successfully deleted
I20240502 07:58:46.607401 22979 consensus_queue.cc:237] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:46.608769 22847 raft_consensus.cc:1261] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:46.608767 22584 raft_consensus.cc:1261] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:46.609087 25670 consensus_queue.cc:1035] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 07:58:46.609242 22915 consensus_peers.cc:485] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): Couldn't send request to peer 4654231dfe994c1a9d2e37d9c3d75699. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 28d5d20938a4472caf029c8d1ac89b82. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:46.609402 25670 consensus_queue.cc:1035] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:46.610689 27333 raft_consensus.cc:2932] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:46.611135 22847 raft_consensus.cc:2932] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:46.613797 22444 catalog_manager.cc:5484] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:46.613862 22584 raft_consensus.cc:2932] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:46.622334 22847 consensus_queue.cc:237] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:46.623559 22715 raft_consensus.cc:1261] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:46.623610 22979 raft_consensus.cc:1261] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:46.623942 26036 consensus_queue.cc:1035] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:46.624112 26079 consensus_queue.cc:1035] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 07:58:46.624293 22781 consensus_peers.cc:485] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): Couldn't send request to peer 6d02017318a2461796d431962fe6a537. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4363e32404304568b94f615e4e29f883. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:46.625803 26079 raft_consensus.cc:2932] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:46.626176 22979 raft_consensus.cc:2932] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:46.626217 22715 raft_consensus.cc:2932] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:46.628760 22444 catalog_manager.cc:5484] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:46.632745 26697 raft_consensus.cc:482] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:46.632843 26697 raft_consensus.cc:504] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:46.633213 26697 leader_election.cc:288] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:46.633550 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "482102b9b3214892a593eb48d1cae943" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:46.633685 25135 raft_consensus.cc:2454] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:46.633769 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "482102b9b3214892a593eb48d1cae943" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:46.633921 24871 raft_consensus.cc:2454] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:46.634033 24939 leader_election.cc:302] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:46.634160 26697 raft_consensus.cc:2781] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:46.634212 26697 raft_consensus.cc:482] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:46.634244 26697 raft_consensus.cc:3037] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:46.635103 26697 raft_consensus.cc:504] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:46.635442 26697 leader_election.cc:288] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:46.635743 27134 raft_consensus.cc:1050] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e: attempting to promote NON_VOTER 7db4178eed174225bfb24f5f9445ce88 to VOTER
I20240502 07:58:46.635921 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "482102b9b3214892a593eb48d1cae943" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:46.636026 25135 raft_consensus.cc:3037] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:46.636161 27134 consensus_queue.cc:237] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } }
I20240502 07:58:46.636850 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "482102b9b3214892a593eb48d1cae943" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:46.636914 25135 raft_consensus.cc:2454] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:46.636952 24871 raft_consensus.cc:3037] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:46.637384 22847 raft_consensus.cc:1261] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:46.637434 25266 raft_consensus.cc:1261] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:46.637703 26696 consensus_queue.cc:1035] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:46.637933 24871 raft_consensus.cc:2454] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:46.638008 26696 consensus_queue.cc:1035] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:46.638155 25135 raft_consensus.cc:1261] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:46.638198 24936 leader_election.cc:302] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:46.638334 26697 raft_consensus.cc:2781] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:46.638415 26697 raft_consensus.cc:686] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Becoming Leader. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Running, Role: LEADER
I20240502 07:58:46.638536 26697 consensus_queue.cc:237] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:46.639556 27134 consensus_queue.cc:1035] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:46.640456 22441 catalog_manager.cc:5484] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: term changed from 0 to 1, leader changed from <none> to 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134). New cstate: current_term: 1 leader_uuid: "74f9aa91c2c448f980b512fe2e8d672e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:46.641216 26696 raft_consensus.cc:2932] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:46.642259 22847 consensus_queue.cc:237] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:46.642912 25266 raft_consensus.cc:2932] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:46.643304 25135 raft_consensus.cc:2932] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:46.643929 22441 catalog_manager.cc:5484] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "74f9aa91c2c448f980b512fe2e8d672e" committed_config { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:46.644548 22715 raft_consensus.cc:1261] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:46.645821 22584 raft_consensus.cc:1261] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:46.646077 22781 consensus_peers.cc:485] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): Couldn't send request to peer 6d02017318a2461796d431962fe6a537. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ee06ff655f9c46c1bb06af2dd539fde7. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:46.646183 26284 consensus_queue.cc:1035] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:46.646651 25003 consensus_queue.cc:237] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } }
I20240502 07:58:46.647328 22845 raft_consensus.cc:2932] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:46.649158 26284 consensus_queue.cc:1035] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:46.650969 25135 raft_consensus.cc:1261] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:46.650977 22845 raft_consensus.cc:1261] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:46.651333 26697 consensus_queue.cc:1035] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:46.651518 26284 raft_consensus.cc:2932] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:46.651754 27134 consensus_queue.cc:1035] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:46.651630 22715 raft_consensus.cc:2932] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:46.653542 22584 raft_consensus.cc:2932] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:46.654148 22441 catalog_manager.cc:5484] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:46.655287 26696 raft_consensus.cc:2932] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:46.656114 25135 raft_consensus.cc:2932] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:46.656980 22430 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 28832d9e18ea41b194cf33d8c7971e23 with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:46.658455 22845 raft_consensus.cc:2932] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:46.658568 22444 catalog_manager.cc:5484] T 28832d9e18ea41b194cf33d8c7971e23 P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: config changed from index 3 to 4, VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) evicted. New cstate: current_term: 1 leader_uuid: "74f9aa91c2c448f980b512fe2e8d672e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:46.661648 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet 28832d9e18ea41b194cf33d8c7971e23 with delete_type TABLET_DATA_TOMBSTONED (TS 7eced220cf1642f3873fc385b6191e69 not found in new config with opid_index 4) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:46.661901 26918 tablet_replica.cc:330] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:46.662082 26918 raft_consensus.cc:2229] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:46.662256 26918 raft_consensus.cc:2258] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:46.662948 26918 ts_tablet_manager.cc:1875] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:46.664824 26918 ts_tablet_manager.cc:1888] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.3
I20240502 07:58:46.664883 26918 log.cc:1196] T 28832d9e18ea41b194cf33d8c7971e23 P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/28832d9e18ea41b194cf33d8c7971e23
I20240502 07:58:46.665321 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet 28832d9e18ea41b194cf33d8c7971e23 (table rebalancer_extra_table_0 [id=3913f9e0fdc04fa2ab31379c9048ad9a]) successfully deleted
I20240502 07:58:46.751376 22441 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:46.763767 25135 raft_consensus.cc:1261] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:46.764438 22697 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 07:58:46.764438 24984 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 07:58:46.766620 22959 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 07:58:46.766955 26697 consensus_queue.cc:1035] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:46.768419 24871 raft_consensus.cc:1261] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:46.769140 26697 consensus_queue.cc:1035] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:46.769201 24852 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:46.771582 22441 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet 482102b9b3214892a593eb48d1cae943 (table rebalancer_dynamic_table [id=aa5a896c477a4ab48ad2be2757181461]). Expected version 1 got 0
I20240502 07:58:46.772228 25246 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 07:58:46.774797 22444 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 482102b9b3214892a593eb48d1cae943 (table rebalancer_dynamic_table [id=aa5a896c477a4ab48ad2be2757181461]). Expected version 1 got 0
I20240502 07:58:46.777300 25113 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:46.785071 22827 tablet_service.cc:1429] Tablet server has 11 leaders and 0 scanners
I20240502 07:58:46.789011 22444 catalog_manager.cc:5576] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857) does not have the latest schema for tablet 482102b9b3214892a593eb48d1cae943 (table rebalancer_dynamic_table [id=aa5a896c477a4ab48ad2be2757181461]). Expected version 1 got 0
I20240502 07:58:46.790575 27482 tablet.cc:1722] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:46.791010 27482 tablet.cc:1620] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:46.796614 22441 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet 482102b9b3214892a593eb48d1cae943 (table rebalancer_dynamic_table [id=aa5a896c477a4ab48ad2be2757181461]). Expected version 1 got 0
I20240502 07:58:46.798660 22566 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 07:58:46.799980 27318 consensus_queue.cc:1035] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:46.801838 22441 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=aa5a896c477a4ab48ad2be2757181461] alter complete (version 1)
I20240502 07:58:46.804540 22444 catalog_manager.cc:5576] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857) does not have the latest schema for tablet 482102b9b3214892a593eb48d1cae943 (table rebalancer_dynamic_table [id=aa5a896c477a4ab48ad2be2757181461]). Expected version 1 got 0
I20240502 07:58:46.805105 22436 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 482102b9b3214892a593eb48d1cae943 (table rebalancer_dynamic_table [id=aa5a896c477a4ab48ad2be2757181461]). Expected version 1 got 0
I20240502 07:58:46.808902 27483 tablet.cc:1722] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:46.809303 27483 tablet.cc:1620] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:46.813084 27481 tablet.cc:1722] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:46.813375 27481 tablet.cc:1620] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e: MemRowSet was empty: no flush needed.
I20240502 07:58:46.815963 27002 consensus_queue.cc:1035] T ed2b3bfe89b24d38b2481bb150b3106a P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:46.856420 25266 consensus_queue.cc:237] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 07:58:46.857813 24871 raft_consensus.cc:1261] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:46.858063 25003 raft_consensus.cc:1261] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:46.858202 27002 consensus_queue.cc:1035] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
W20240502 07:58:46.858470 25200 consensus_peers.cc:485] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 -> Peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): Couldn't send request to peer 02a92e1c084b4308b9e855a4cc2c295f. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4e687b2e36574856a45752104ce83953. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:46.858541 27002 consensus_queue.cc:1035] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:46.859952 27002 raft_consensus.cc:2932] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 07:58:46.860384 24871 raft_consensus.cc:2932] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 07:58:46.860882 25003 raft_consensus.cc:2932] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 07:58:46.863255 22436 catalog_manager.cc:5484] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537 reported cstate change: config changed from index -1 to 2, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 07:58:46.870671 24871 consensus_queue.cc:237] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 07:58:46.872135 22584 raft_consensus.cc:1261] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:46.872931 27338 consensus_queue.cc:1035] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:46.872989 25003 raft_consensus.cc:1261] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:46.874048 27338 consensus_queue.cc:1035] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
W20240502 07:58:46.875901 24808 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): Couldn't send request to peer 7db4178eed174225bfb24f5f9445ce88. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a071d3a1489e4d97ae6ceea4b0478459. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:46.876001 27110 raft_consensus.cc:2932] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:46.876340 22584 raft_consensus.cc:2932] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:46.877087 25003 raft_consensus.cc:2932] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:46.878460 22441 catalog_manager.cc:5484] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 reported cstate change: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:46.886611 25266 consensus_queue.cc:237] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 07:58:46.888058 25134 raft_consensus.cc:1261] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:46.888429 27495 consensus_queue.cc:1035] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:46.888522 24871 raft_consensus.cc:1261] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
W20240502 07:58:46.888604 25199 consensus_peers.cc:485] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 -> Peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): Couldn't send request to peer 725969d490a04fdfb3ba890c3f5096e3. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cfcaa9ee1fbe48ae91796b184ecaf2a3. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:46.888844 27002 consensus_queue.cc:1035] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:46.890296 27495 raft_consensus.cc:2932] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } } }
I20240502 07:58:46.891134 24871 raft_consensus.cc:2932] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } } }
I20240502 07:58:46.891240 25134 raft_consensus.cc:2932] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } } }
I20240502 07:58:46.893628 22436 catalog_manager.cc:5484] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index -1 to 2, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } } }
I20240502 07:58:46.900818 25003 raft_consensus.cc:1203] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:46.906852 25003 consensus_queue.cc:237] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:46.908152 22441 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:46.908622 24871 raft_consensus.cc:1261] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:46.909214 25134 raft_consensus.cc:1261] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:46.910005 27484 consensus_queue.cc:1035] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:46.911722 27484 consensus_queue.cc:1035] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:46.914294 27134 raft_consensus.cc:2932] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index -1 to 3, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:46.915599 27481 tablet.cc:1722] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:46.915905 27481 tablet.cc:1620] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e: MemRowSet was empty: no flush needed.
I20240502 07:58:46.916710 25134 raft_consensus.cc:2932] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index -1 to 3, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:46.916743 24871 raft_consensus.cc:2932] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index -1 to 3, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:46.917966 27483 tablet.cc:1722] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:46.918205 27483 tablet.cc:1620] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:46.919121 27482 tablet.cc:1722] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:46.919093 22441 catalog_manager.cc:5484] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537 reported cstate change: config changed from index -1 to 3, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New cstate: current_term: 1 leader_uuid: "74f9aa91c2c448f980b512fe2e8d672e" committed_config { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:46.919297 22441 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet 482102b9b3214892a593eb48d1cae943 (table rebalancer_dynamic_table [id=aa5a896c477a4ab48ad2be2757181461]). Expected version 2 got 1
I20240502 07:58:46.919364 27482 tablet.cc:1620] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
W20240502 07:58:46.924726 24938 consensus_peers.cc:485] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e -> Peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Couldn't send request to peer 7eced220cf1642f3873fc385b6191e69. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 482102b9b3214892a593eb48d1cae943. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:46.925334 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=aa5a896c477a4ab48ad2be2757181461] alter complete (version 2)
I20240502 07:58:46.926012 22444 catalog_manager.cc:5576] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857) does not have the latest schema for tablet 482102b9b3214892a593eb48d1cae943 (table rebalancer_dynamic_table [id=aa5a896c477a4ab48ad2be2757181461]). Expected version 2 got 1
I20240502 07:58:46.935320 22715 consensus_queue.cc:237] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:46.936642 22979 raft_consensus.cc:1261] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:46.936745 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cceaa8887081401dbc9154d265004c7b. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:46.936779 22845 raft_consensus.cc:1261] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:46.937105 27303 consensus_queue.cc:1035] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:46.937551 27303 consensus_queue.cc:1035] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:46.938673 26995 raft_consensus.cc:2932] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:46.939003 22979 raft_consensus.cc:2932] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:46.941792 22441 catalog_manager.cc:5484] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:46.944445 22845 raft_consensus.cc:2932] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:46.949546 22845 consensus_queue.cc:237] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } }
I20240502 07:58:46.951145 25134 raft_consensus.cc:1261] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:46.951421 25266 raft_consensus.cc:1261] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
W20240502 07:58:46.951714 22784 consensus_peers.cc:485] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 -> Peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): Couldn't send request to peer 8bea0fe8d8044a2685f92f7a10c9cf85. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 66ac69e448574c7ea5101014e0dd425d. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:46.951828 26079 consensus_queue.cc:1035] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:46.952024 26036 consensus_queue.cc:1035] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:46.953627 26079 raft_consensus.cc:2932] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } } }
I20240502 07:58:46.954113 25266 raft_consensus.cc:2932] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } } }
I20240502 07:58:46.955571 25134 raft_consensus.cc:2932] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } } }
I20240502 07:58:46.957022 22436 catalog_manager.cc:5484] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 2, NON_VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) added. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:46.963663 22979 consensus_queue.cc:237] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:46.964952 22715 raft_consensus.cc:1261] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 07:58:46.965274 27310 consensus_queue.cc:1035] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:46.965368 25266 raft_consensus.cc:1261] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 07:58:46.965584 22915 consensus_peers.cc:485] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): Couldn't send request to peer 4654231dfe994c1a9d2e37d9c3d75699. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cfff02438656430686be0f03f5c0b8ab. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:46.965729 25670 consensus_queue.cc:1035] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:46.966827 26128 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:46.967890 25265 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:46.969028 22715 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:46.969512 22444 catalog_manager.cc:5484] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 6 to 7, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:46.977034 22584 consensus_queue.cc:237] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 07:58:46.978169 24872 raft_consensus.cc:1261] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:46.978224 22979 raft_consensus.cc:1261] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
W20240502 07:58:46.978384 22522 consensus_peers.cc:485] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): Couldn't send request to peer 7db4178eed174225bfb24f5f9445ce88. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 415879c2eee74932b9ac45282f6dd0fb. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:46.978536 25987 consensus_queue.cc:1035] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:46.978667 27023 consensus_queue.cc:1035] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:46.980301 27136 raft_consensus.cc:2932] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:46.980728 24871 raft_consensus.cc:2932] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:46.980734 22979 raft_consensus.cc:2932] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:46.982955 22436 catalog_manager.cc:5484] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537 reported cstate change: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:46.989866 22715 consensus_queue.cc:237] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:46.991118 25265 raft_consensus.cc:1261] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:46.991178 24871 raft_consensus.cc:1261] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
W20240502 07:58:46.991218 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c3a7b2e52ba641e993cf1d5dc9724473. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:46.991533 26995 consensus_queue.cc:1035] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:46.991603 27338 consensus_queue.cc:1035] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:46.991690 27303 consensus_queue.cc:1035] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:46.993388 27303 raft_consensus.cc:2932] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:46.993973 22435 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_18" } modify_external_catalogs: true
I20240502 07:58:46.994071 22435 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalance_test_table_18" } modify_external_catalogs: true
I20240502 07:58:46.994433 27110 consensus_queue.cc:1035] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:46.994980 25265 raft_consensus.cc:2932] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:46.996596 22436 catalog_manager.cc:5484] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index -1 to 2, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:46.998833 22435 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 68aff050f5f7483ca821f1d28b04c91c
I20240502 07:58:46.999174 22435 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 45499d82e94c47b8b398c159df9fd49b
I20240502 07:58:46.999395 22435 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet b583c8775a874ad1a89c82c00c05bbbd
I20240502 07:58:46.999598 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet 68aff050f5f7483ca821f1d28b04c91c with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:46.999753 27126 tablet_replica.cc:330] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:46.999890 27126 raft_consensus.cc:2229] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:47.000008 27126 raft_consensus.cc:2258] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.000278 24871 raft_consensus.cc:2932] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:47.000506 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet 45499d82e94c47b8b398c159df9fd49b with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:47.000561 27126 ts_tablet_manager.cc:1875] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:47.000771 26920 tablet_replica.cc:330] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:47.000903 26920 raft_consensus.cc:2229] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:47.001101 26920 raft_consensus.cc:2258] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.001570 26920 ts_tablet_manager.cc:1875] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:47.001698 24984 tablet_service.cc:1514] Processing DeleteTablet for tablet b583c8775a874ad1a89c82c00c05bbbd with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:42878
I20240502 07:58:47.001724 22564 tablet_service.cc:1514] Processing DeleteTablet for tablet b583c8775a874ad1a89c82c00c05bbbd with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:47.001879 26956 tablet_replica.cc:330] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e: stopping tablet replica
I20240502 07:58:47.001982 26956 raft_consensus.cc:2229] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:47.002111 26956 raft_consensus.cc:2258] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.002601 27126 ts_tablet_manager.cc:1888] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:47.002658 27126 log.cc:1196] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/68aff050f5f7483ca821f1d28b04c91c
I20240502 07:58:47.002807 26956 ts_tablet_manager.cc:1875] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:47.002902 27126 ts_tablet_manager.cc:1909] T 68aff050f5f7483ca821f1d28b04c91c P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:47.003669 26920 ts_tablet_manager.cc:1888] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:47.003727 26920 log.cc:1196] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/45499d82e94c47b8b398c159df9fd49b
I20240502 07:58:47.003785 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 68aff050f5f7483ca821f1d28b04c91c with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:47.003937 25468 tablet_replica.cc:330] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:47.004007 26920 ts_tablet_manager.cc:1909] T 45499d82e94c47b8b398c159df9fd49b P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:47.004051 25468 raft_consensus.cc:2229] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:47.004240 25468 raft_consensus.cc:2258] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.004554 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet b583c8775a874ad1a89c82c00c05bbbd with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:47.004699 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 45499d82e94c47b8b398c159df9fd49b (table rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]) successfully deleted
I20240502 07:58:47.004989 25468 ts_tablet_manager.cc:1875] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:47.005029 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet 45499d82e94c47b8b398c159df9fd49b with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:47.005398 26956 ts_tablet_manager.cc:1888] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:47.005470 26956 log.cc:1196] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/wals/b583c8775a874ad1a89c82c00c05bbbd
I20240502 07:58:47.005728 26956 ts_tablet_manager.cc:1909] T b583c8775a874ad1a89c82c00c05bbbd P 74f9aa91c2c448f980b512fe2e8d672e: Deleting consensus metadata
I20240502 07:58:47.005766 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet 68aff050f5f7483ca821f1d28b04c91c with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:47.005981 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet 45499d82e94c47b8b398c159df9fd49b with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:46 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:47.006034 27566 tablet_replica.cc:330] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:47.006145 27566 raft_consensus.cc:2229] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:47.006345 22430 catalog_manager.cc:4830] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): tablet b583c8775a874ad1a89c82c00c05bbbd (table rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]) successfully deleted
I20240502 07:58:47.006544 27566 raft_consensus.cc:2258] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.006757 25468 ts_tablet_manager.cc:1888] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:47.006820 25468 log.cc:1196] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/b583c8775a874ad1a89c82c00c05bbbd
I20240502 07:58:47.007038 25468 ts_tablet_manager.cc:1909] T b583c8775a874ad1a89c82c00c05bbbd P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:47.007087 27566 ts_tablet_manager.cc:1875] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:47.007586 25468 tablet_replica.cc:330] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:47.007684 25468 raft_consensus.cc:2229] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:47.007664 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet b583c8775a874ad1a89c82c00c05bbbd (table rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]) successfully deleted
I20240502 07:58:47.007831 25468 raft_consensus.cc:2258] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.008195 25468 ts_tablet_manager.cc:1875] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:47.008769 26918 tablet_replica.cc:330] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:47.008873 26918 raft_consensus.cc:2229] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:47.008977 26918 raft_consensus.cc:2258] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.009445 26918 ts_tablet_manager.cc:1875] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:47.009482 27566 ts_tablet_manager.cc:1888] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:47.009531 27566 log.cc:1196] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/68aff050f5f7483ca821f1d28b04c91c
I20240502 07:58:47.009757 27566 ts_tablet_manager.cc:1909] T 68aff050f5f7483ca821f1d28b04c91c P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:47.010195 27566 tablet_replica.cc:330] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:47.010241 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 68aff050f5f7483ca821f1d28b04c91c (table rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]) successfully deleted
I20240502 07:58:47.010286 27566 raft_consensus.cc:2229] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:47.010433 27566 raft_consensus.cc:2258] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.010891 27566 ts_tablet_manager.cc:1875] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:47.011384 26918 ts_tablet_manager.cc:1888] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:47.011440 26918 log.cc:1196] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/45499d82e94c47b8b398c159df9fd49b
I20240502 07:58:47.011696 26918 ts_tablet_manager.cc:1909] T 45499d82e94c47b8b398c159df9fd49b P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:58:47.012434 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet 45499d82e94c47b8b398c159df9fd49b (table rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]) successfully deleted
I20240502 07:58:47.012832 27126 tablet_replica.cc:330] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:47.012916 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 68aff050f5f7483ca821f1d28b04c91c (table rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]) successfully deleted
I20240502 07:58:47.012959 27126 raft_consensus.cc:2229] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:47.012995 27566 ts_tablet_manager.cc:1888] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:47.013063 27566 log.cc:1196] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/45499d82e94c47b8b398c159df9fd49b
I20240502 07:58:47.013119 27126 raft_consensus.cc:2258] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.013337 27566 ts_tablet_manager.cc:1909] T 45499d82e94c47b8b398c159df9fd49b P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:47.013732 27126 ts_tablet_manager.cc:1875] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:47.013913 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 45499d82e94c47b8b398c159df9fd49b (table rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]) successfully deleted
I20240502 07:58:47.014632 25468 ts_tablet_manager.cc:1888] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20240502 07:58:47.014704 25468 log.cc:1196] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/68aff050f5f7483ca821f1d28b04c91c
I20240502 07:58:47.014950 25468 ts_tablet_manager.cc:1909] T 68aff050f5f7483ca821f1d28b04c91c P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:47.015543 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 68aff050f5f7483ca821f1d28b04c91c (table rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]) successfully deleted
I20240502 07:58:47.015744 27126 ts_tablet_manager.cc:1888] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.6
I20240502 07:58:47.015877 27126 log.cc:1196] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/b583c8775a874ad1a89c82c00c05bbbd
I20240502 07:58:47.016395 27126 ts_tablet_manager.cc:1909] T b583c8775a874ad1a89c82c00c05bbbd P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:47.016983 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet b583c8775a874ad1a89c82c00c05bbbd (table rebalance_test_table_18 [id=fc4b50ece08345c7a84625ae0b48a04f]) successfully deleted
I20240502 07:58:47.020411 22845 consensus_queue.cc:237] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:47.021687 22979 raft_consensus.cc:1261] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:47.021726 22584 raft_consensus.cc:1261] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:47.021960 22784 consensus_peers.cc:485] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 -> Peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): Couldn't send request to peer 4654231dfe994c1a9d2e37d9c3d75699. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fcbab4ec41a74ff8b0c564ac7a0b044e. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:47.022006 25532 consensus_queue.cc:1035] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:47.022156 26036 consensus_queue.cc:1035] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:47.023679 26036 raft_consensus.cc:2932] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:47.024096 22979 raft_consensus.cc:2932] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:47.024192 22584 raft_consensus.cc:2932] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:47.026429 22435 catalog_manager.cc:5484] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:47.029157 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:47.034395 27481 tablet.cc:1722] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:47.034772 27482 tablet.cc:1722] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:47.035778 27303 consensus_queue.cc:1035] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:47.037555 27483 tablet.cc:1722] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:47.039774 22715 consensus_queue.cc:237] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:47.041055 25003 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 07:58:47.041129 22435 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=aa5a896c477a4ab48ad2be2757181461] alter complete (version 3)
I20240502 07:58:47.041185 22584 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 07:58:47.041702 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Couldn't send request to peer 7eced220cf1642f3873fc385b6191e69. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e25ba5cc77334a3595a4575fa98476f0. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:47.041889 27303 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:47.042052 27301 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:47.043704 27301 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:47.044862 27303 consensus_queue.cc:1035] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:47.045064 22586 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:47.045980 25003 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:47.048738 22436 catalog_manager.cc:5484] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:47.074034 27599 ts_tablet_manager.cc:902] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:47.074685 27599 tablet_copy_client.cc:308] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:47.075116 22999 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet 28d5d20938a4472caf029c8d1ac89b82 from peer 4654231dfe994c1a9d2e37d9c3d75699 ({username='slave'} at 127.16.85.135:41731)
I20240502 07:58:47.075207 22999 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet 28d5d20938a4472caf029c8d1ac89b82 from peer 4654231dfe994c1a9d2e37d9c3d75699 at {username='slave'} at 127.16.85.135:41731: session id = 4654231dfe994c1a9d2e37d9c3d75699-28d5d20938a4472caf029c8d1ac89b82
I20240502 07:58:47.075855 22999 tablet_copy_source_session.cc:205] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:47.076753 27599 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 28d5d20938a4472caf029c8d1ac89b82. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.078802 27599 tablet_copy_client.cc:783] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:47.078955 27599 tablet_copy_client.cc:647] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:47.080219 27599 tablet_copy_client.cc:523] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:47.081591 27599 tablet_bootstrap.cc:492] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:47.104825 27599 tablet_bootstrap.cc:492] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:47.105109 27599 tablet_bootstrap.cc:492] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap complete.
I20240502 07:58:47.105275 27599 ts_tablet_manager.cc:1367] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.024s user 0.020s sys 0.003s
I20240502 07:58:47.105700 27599 raft_consensus.cc:348] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:47.105856 27599 raft_consensus.cc:729] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: LEARNER
I20240502 07:58:47.105957 27599 consensus_queue.cc:260] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:47.106271 27599 ts_tablet_manager.cc:1398] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:47.106686 22999 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-28d5d20938a4472caf029c8d1ac89b82 received from {username='slave'} at 127.16.85.135:41731
I20240502 07:58:47.106765 22999 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-28d5d20938a4472caf029c8d1ac89b82 on tablet 28d5d20938a4472caf029c8d1ac89b82 with peer 4654231dfe994c1a9d2e37d9c3d75699
I20240502 07:58:47.118999 27624 ts_tablet_manager.cc:902] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537: Initiating tablet copy from peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:47.119525 27624 tablet_copy_client.cc:308] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.131:44055
I20240502 07:58:47.119997 22868 tablet_copy_service.cc:140] P 7db4178eed174225bfb24f5f9445ce88: Received BeginTabletCopySession request for tablet 4363e32404304568b94f615e4e29f883 from peer 6d02017318a2461796d431962fe6a537 ({username='slave'} at 127.16.85.133:51347)
I20240502 07:58:47.120090 22868 tablet_copy_service.cc:161] P 7db4178eed174225bfb24f5f9445ce88: Beginning new tablet copy session on tablet 4363e32404304568b94f615e4e29f883 from peer 6d02017318a2461796d431962fe6a537 at {username='slave'} at 127.16.85.133:51347: session id = 6d02017318a2461796d431962fe6a537-4363e32404304568b94f615e4e29f883
I20240502 07:58:47.120925 22868 tablet_copy_source_session.cc:205] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:47.122148 27624 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4363e32404304568b94f615e4e29f883. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.124321 27624 tablet_copy_client.cc:783] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:47.124480 27624 tablet_copy_client.cc:647] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:47.125476 27624 tablet_copy_client.cc:523] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:47.126657 27624 tablet_bootstrap.cc:492] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:47.142480 22957 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 07:58:47.143831 22440 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:47.143971 22440 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:47.146009 22440 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 4 replicas of tablet 482102b9b3214892a593eb48d1cae943
I20240502 07:58:47.146567 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet 482102b9b3214892a593eb48d1cae943 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:47 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:47.146975 25113 tablet_service.cc:1514] Processing DeleteTablet for tablet 482102b9b3214892a593eb48d1cae943 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:47 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:47.147819 22440 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_12"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:47.148840 22827 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 07:58:47.148981 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet 482102b9b3214892a593eb48d1cae943 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:47 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:47.149032 24983 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 07:58:47.149112 24984 tablet_service.cc:1514] Processing DeleteTablet for tablet 482102b9b3214892a593eb48d1cae943 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:47 UTC) from {username='slave'} at 127.0.0.1:42878
W20240502 07:58:47.149718 22430 catalog_manager.cc:4797] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): delete failed for tablet 482102b9b3214892a593eb48d1cae943 because the tablet was not found. No further retry: Not found: Tablet not found: 482102b9b3214892a593eb48d1cae943
I20240502 07:58:47.151713 26956 tablet_replica.cc:330] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e: stopping tablet replica
I20240502 07:58:47.151846 26956 raft_consensus.cc:2229] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:47.152086 26956 raft_consensus.cc:2258] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.153232 27624 tablet_bootstrap.cc:492] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:47.153674 27624 tablet_bootstrap.cc:492] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537: Bootstrap complete.
I20240502 07:58:47.154363 26956 ts_tablet_manager.cc:1875] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:47.154729 27642 tablet_replica.cc:330] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:47.154855 27642 raft_consensus.cc:2229] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:47.154994 27642 raft_consensus.cc:2258] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.155184 27624 ts_tablet_manager.cc:1367] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.029s user 0.019s sys 0.004s
I20240502 07:58:47.155581 27624 raft_consensus.cc:348] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:47.155699 27624 raft_consensus.cc:729] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: LEARNER
I20240502 07:58:47.155819 27624 consensus_queue.cc:260] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:47.156180 27624 ts_tablet_manager.cc:1398] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.156311 27642 ts_tablet_manager.cc:1875] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:47.157095 26956 ts_tablet_manager.cc:1888] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.5
I20240502 07:58:47.157179 26956 log.cc:1196] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/wals/482102b9b3214892a593eb48d1cae943
I20240502 07:58:47.157447 26956 ts_tablet_manager.cc:1909] T 482102b9b3214892a593eb48d1cae943 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting consensus metadata
I20240502 07:58:47.158138 22430 catalog_manager.cc:4830] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): tablet 482102b9b3214892a593eb48d1cae943 (table rebalancer_dynamic_table_new_name [id=aa5a896c477a4ab48ad2be2757181461]) successfully deleted
I20240502 07:58:47.158579 22868 tablet_copy_service.cc:342] P 7db4178eed174225bfb24f5f9445ce88: Request end of tablet copy session 6d02017318a2461796d431962fe6a537-4363e32404304568b94f615e4e29f883 received from {username='slave'} at 127.16.85.133:51347
I20240502 07:58:47.158658 22868 tablet_copy_service.cc:433] P 7db4178eed174225bfb24f5f9445ce88: ending tablet copy session 6d02017318a2461796d431962fe6a537-4363e32404304568b94f615e4e29f883 on tablet 4363e32404304568b94f615e4e29f883 with peer 6d02017318a2461796d431962fe6a537
I20240502 07:58:47.159262 27642 ts_tablet_manager.cc:1888] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.5
I20240502 07:58:47.159363 27642 log.cc:1196] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/482102b9b3214892a593eb48d1cae943
I20240502 07:58:47.159766 27642 ts_tablet_manager.cc:1909] T 482102b9b3214892a593eb48d1cae943 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:47.160230 22697 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 07:58:47.160323 24983 tablet_service.cc:1467] Processing CreateTablet for tablet d2eebf9aaa2c4f92a4fefbe5bb6feb0c (DEFAULT_TABLE table=rebalancer_extra_table_12 [id=c6e87fd7585e4839a6a17f6b144be0a4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.160369 24984 tablet_service.cc:1467] Processing CreateTablet for tablet 7eb1b4bcf48b4cbea5cfb4dd64de2c70 (DEFAULT_TABLE table=rebalancer_extra_table_12 [id=c6e87fd7585e4839a6a17f6b144be0a4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.160583 24983 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d2eebf9aaa2c4f92a4fefbe5bb6feb0c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.160588 25246 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 07:58:47.160708 24984 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7eb1b4bcf48b4cbea5cfb4dd64de2c70. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.161170 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 70ab3b4bafee41d883c87d30b63f756f (DEFAULT_TABLE table=rebalancer_extra_table_12 [id=c6e87fd7585e4839a6a17f6b144be0a4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.161422 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 70ab3b4bafee41d883c87d30b63f756f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.161861 25113 tablet_service.cc:1467] Processing CreateTablet for tablet d2eebf9aaa2c4f92a4fefbe5bb6feb0c (DEFAULT_TABLE table=rebalancer_extra_table_12 [id=c6e87fd7585e4839a6a17f6b144be0a4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.162060 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d2eebf9aaa2c4f92a4fefbe5bb6feb0c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.162644 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 70ab3b4bafee41d883c87d30b63f756f (DEFAULT_TABLE table=rebalancer_extra_table_12 [id=c6e87fd7585e4839a6a17f6b144be0a4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.162899 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 70ab3b4bafee41d883c87d30b63f756f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.164019 27644 tablet_bootstrap.cc:492] T 70ab3b4bafee41d883c87d30b63f756f P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:47.161170 24851 tablet_service.cc:1467] Processing CreateTablet for tablet d2eebf9aaa2c4f92a4fefbe5bb6feb0c (DEFAULT_TABLE table=rebalancer_extra_table_12 [id=c6e87fd7585e4839a6a17f6b144be0a4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.165047 24851 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d2eebf9aaa2c4f92a4fefbe5bb6feb0c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.165045 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 7eb1b4bcf48b4cbea5cfb4dd64de2c70 (DEFAULT_TABLE table=rebalancer_extra_table_12 [id=c6e87fd7585e4839a6a17f6b144be0a4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.165079 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 70ab3b4bafee41d883c87d30b63f756f (DEFAULT_TABLE table=rebalancer_extra_table_12 [id=c6e87fd7585e4839a6a17f6b144be0a4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.165217 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7eb1b4bcf48b4cbea5cfb4dd64de2c70. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.165311 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 70ab3b4bafee41d883c87d30b63f756f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.167359 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet 482102b9b3214892a593eb48d1cae943 (table rebalancer_dynamic_table_new_name [id=aa5a896c477a4ab48ad2be2757181461]) successfully deleted
I20240502 07:58:47.167671 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 7eb1b4bcf48b4cbea5cfb4dd64de2c70 (DEFAULT_TABLE table=rebalancer_extra_table_12 [id=c6e87fd7585e4839a6a17f6b144be0a4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.167874 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7eb1b4bcf48b4cbea5cfb4dd64de2c70. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.167899 27644 tablet_bootstrap.cc:654] T 70ab3b4bafee41d883c87d30b63f756f P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.168406 27647 tablet_bootstrap.cc:492] T 70ab3b4bafee41d883c87d30b63f756f P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:47.169396 27647 tablet_bootstrap.cc:654] T 70ab3b4bafee41d883c87d30b63f756f P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.171890 27645 tablet_bootstrap.cc:492] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:47.173247 27646 tablet_bootstrap.cc:492] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:47.174124 27645 tablet_bootstrap.cc:654] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.174742 27646 tablet_bootstrap.cc:654] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.175232 27641 tablet_replica.cc:330] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:47.175312 27645 tablet_bootstrap.cc:492] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:47.175360 27641 raft_consensus.cc:2229] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:47.175382 27645 ts_tablet_manager.cc:1367] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.002s
I20240502 07:58:47.175477 27641 raft_consensus.cc:2258] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.175747 27645 raft_consensus.cc:348] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.175829 27645 raft_consensus.cc:374] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.175853 27645 raft_consensus.cc:729] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.175946 27645 consensus_queue.cc:260] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.176138 27641 ts_tablet_manager.cc:1875] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:47.177266 27645 ts_tablet_manager.cc:1398] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:47.178596 27641 ts_tablet_manager.cc:1888] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.5
I20240502 07:58:47.178658 27641 log.cc:1196] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/482102b9b3214892a593eb48d1cae943
I20240502 07:58:47.178926 27641 ts_tablet_manager.cc:1909] T 482102b9b3214892a593eb48d1cae943 P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:47.179237 25113 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 07:58:47.179534 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet 482102b9b3214892a593eb48d1cae943 (table rebalancer_dynamic_table_new_name [id=aa5a896c477a4ab48ad2be2757181461]) successfully deleted
I20240502 07:58:47.179960 27644 tablet_bootstrap.cc:492] T 70ab3b4bafee41d883c87d30b63f756f P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:47.180043 27644 ts_tablet_manager.cc:1367] T 70ab3b4bafee41d883c87d30b63f756f P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.016s user 0.002s sys 0.000s
I20240502 07:58:47.180438 27644 raft_consensus.cc:348] T 70ab3b4bafee41d883c87d30b63f756f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:47.180524 27644 raft_consensus.cc:374] T 70ab3b4bafee41d883c87d30b63f756f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.180552 27644 raft_consensus.cc:729] T 70ab3b4bafee41d883c87d30b63f756f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.180645 27644 consensus_queue.cc:260] T 70ab3b4bafee41d883c87d30b63f756f P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:47.180925 27644 ts_tablet_manager.cc:1398] T 70ab3b4bafee41d883c87d30b63f756f P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.181067 27644 tablet_bootstrap.cc:492] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:47.182090 27644 tablet_bootstrap.cc:654] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.183513 27644 tablet_bootstrap.cc:492] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:47.183593 27644 ts_tablet_manager.cc:1367] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:47.184019 27644 raft_consensus.cc:348] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:47.184111 27644 raft_consensus.cc:374] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.184146 27644 raft_consensus.cc:729] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.184252 27644 consensus_queue.cc:260] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:47.184516 27644 ts_tablet_manager.cc:1398] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.184520 22566 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 07:58:47.184592 24852 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 07:58:47.184660 27644 tablet_bootstrap.cc:492] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:47.185931 27644 tablet_bootstrap.cc:654] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.186823 27644 tablet_bootstrap.cc:492] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:47.186892 27644 ts_tablet_manager.cc:1367] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.002s
I20240502 07:58:47.187191 27644 raft_consensus.cc:348] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.187258 27644 raft_consensus.cc:374] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.187279 27644 raft_consensus.cc:729] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.187376 27644 consensus_queue.cc:260] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.187552 27644 ts_tablet_manager.cc:1398] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:47.194259 27647 tablet_bootstrap.cc:492] T 70ab3b4bafee41d883c87d30b63f756f P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:47.194404 27647 ts_tablet_manager.cc:1367] T 70ab3b4bafee41d883c87d30b63f756f P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.026s user 0.002s sys 0.000s
I20240502 07:58:47.194857 27647 raft_consensus.cc:348] T 70ab3b4bafee41d883c87d30b63f756f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:47.195075 27647 raft_consensus.cc:374] T 70ab3b4bafee41d883c87d30b63f756f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.195140 27647 raft_consensus.cc:729] T 70ab3b4bafee41d883c87d30b63f756f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.195297 27647 consensus_queue.cc:260] T 70ab3b4bafee41d883c87d30b63f756f P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:47.195616 27647 ts_tablet_manager.cc:1398] T 70ab3b4bafee41d883c87d30b63f756f P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.195806 27647 tablet_bootstrap.cc:492] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:47.196645 27647 tablet_bootstrap.cc:654] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.198014 27646 tablet_bootstrap.cc:492] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:47.198107 27646 ts_tablet_manager.cc:1367] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.025s user 0.002s sys 0.000s
I20240502 07:58:47.198518 27646 raft_consensus.cc:348] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:47.198606 27646 raft_consensus.cc:374] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.198633 27646 raft_consensus.cc:729] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.198742 27646 consensus_queue.cc:260] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:47.199035 27646 ts_tablet_manager.cc:1398] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:47.218482 27647 tablet_bootstrap.cc:492] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:47.218569 27647 ts_tablet_manager.cc:1367] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.023s user 0.002s sys 0.000s
I20240502 07:58:47.218989 27647 raft_consensus.cc:348] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:47.219105 27647 raft_consensus.cc:374] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.219130 27647 raft_consensus.cc:729] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.219230 27647 consensus_queue.cc:260] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:47.219450 27647 ts_tablet_manager.cc:1398] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.220901 27643 tablet_bootstrap.cc:492] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:47.224864 27643 tablet_bootstrap.cc:654] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.225906 27643 tablet_bootstrap.cc:492] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:47.225980 27643 ts_tablet_manager.cc:1367] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:58:47.226451 27643 raft_consensus.cc:348] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:47.226552 27643 raft_consensus.cc:374] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.226577 27643 raft_consensus.cc:729] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.226680 27643 consensus_queue.cc:260] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:47.226902 27643 ts_tablet_manager.cc:1398] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.227032 27643 tablet_bootstrap.cc:492] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:47.227813 27643 tablet_bootstrap.cc:654] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.228863 27643 tablet_bootstrap.cc:492] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:47.228935 27643 ts_tablet_manager.cc:1367] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:47.229296 27643 raft_consensus.cc:348] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.229391 27643 raft_consensus.cc:374] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.229565 27643 raft_consensus.cc:729] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.229723 27643 consensus_queue.cc:260] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.229943 27643 ts_tablet_manager.cc:1398] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:47.236997 27488 raft_consensus.cc:482] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:47.237097 27488 raft_consensus.cc:504] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.237546 27488 leader_election.cc:288] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:47.237805 25003 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d2eebf9aaa2c4f92a4fefbe5bb6feb0c" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e" is_pre_election: true
I20240502 07:58:47.237917 25003 raft_consensus.cc:2454] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:47.238210 25070 leader_election.cc:302] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:47.238368 27488 raft_consensus.cc:2781] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:47.238418 27488 raft_consensus.cc:482] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:47.238449 27488 raft_consensus.cc:3037] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.239267 27488 raft_consensus.cc:504] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.239526 27488 leader_election.cc:288] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:47.239831 25003 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d2eebf9aaa2c4f92a4fefbe5bb6feb0c" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e"
I20240502 07:58:47.239925 25003 raft_consensus.cc:3037] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.240628 25003 raft_consensus.cc:2454] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:47.240947 25070 leader_election.cc:302] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:47.241113 27488 raft_consensus.cc:2781] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:47.241195 27488 raft_consensus.cc:686] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:58:47.241319 27488 consensus_queue.cc:237] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.242964 22442 catalog_manager.cc:5484] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:47.245364 27339 raft_consensus.cc:482] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:47.245469 27339 raft_consensus.cc:504] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:47.245807 27339 leader_election.cc:288] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:47.246150 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7eb1b4bcf48b4cbea5cfb4dd64de2c70" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:47.246280 25266 raft_consensus.cc:2454] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:47.246820 25003 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7eb1b4bcf48b4cbea5cfb4dd64de2c70" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e" is_pre_election: true
I20240502 07:58:47.246914 25003 raft_consensus.cc:2454] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:47.247226 24807 leader_election.cc:302] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:47.247514 27339 raft_consensus.cc:2781] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:47.247561 27339 raft_consensus.cc:482] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:47.247592 27339 raft_consensus.cc:3037] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.247753 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d2eebf9aaa2c4f92a4fefbe5bb6feb0c" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:47.247807 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d2eebf9aaa2c4f92a4fefbe5bb6feb0c" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:47.247869 24871 raft_consensus.cc:2454] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:47.248492 27339 raft_consensus.cc:504] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:47.248844 27339 leader_election.cc:288] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:47.249243 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7eb1b4bcf48b4cbea5cfb4dd64de2c70" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:47.249343 25266 raft_consensus.cc:3037] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.249577 25003 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7eb1b4bcf48b4cbea5cfb4dd64de2c70" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e"
I20240502 07:58:47.249518 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:47.249668 25003 raft_consensus.cc:3037] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.250250 25266 raft_consensus.cc:2454] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:47.250548 25003 raft_consensus.cc:2454] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:47.251773 24807 leader_election.cc:302] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:47.252105 27339 raft_consensus.cc:2781] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:47.252208 27339 raft_consensus.cc:686] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:58:47.252327 27339 consensus_queue.cc:237] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:47.254447 22442 catalog_manager.cc:5484] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:47.254598 24852 tablet_service.cc:1467] Processing CreateTablet for tablet f25bbec445114d098d3af45d67c8a3b8 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=179ac83ba7bd4cd29d0cd00d9e40fdb5]), partition=
I20240502 07:58:47.254645 25113 tablet_service.cc:1467] Processing CreateTablet for tablet f25bbec445114d098d3af45d67c8a3b8 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=179ac83ba7bd4cd29d0cd00d9e40fdb5]), partition=
I20240502 07:58:47.254819 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f25bbec445114d098d3af45d67c8a3b8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.254849 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f25bbec445114d098d3af45d67c8a3b8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.256736 27645 tablet_bootstrap.cc:492] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:47.257509 27645 tablet_bootstrap.cc:654] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.257588 27624 ts_tablet_manager.cc:902] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537: Initiating tablet copy from peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:47.258062 27644 tablet_bootstrap.cc:492] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:47.258101 27624 tablet_copy_client.cc:308] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.131:44055
I20240502 07:58:47.258500 22868 tablet_copy_service.cc:140] P 7db4178eed174225bfb24f5f9445ce88: Received BeginTabletCopySession request for tablet ee06ff655f9c46c1bb06af2dd539fde7 from peer 6d02017318a2461796d431962fe6a537 ({username='slave'} at 127.16.85.133:51347)
I20240502 07:58:47.258589 22868 tablet_copy_service.cc:161] P 7db4178eed174225bfb24f5f9445ce88: Beginning new tablet copy session on tablet ee06ff655f9c46c1bb06af2dd539fde7 from peer 6d02017318a2461796d431962fe6a537 at {username='slave'} at 127.16.85.133:51347: session id = 6d02017318a2461796d431962fe6a537-ee06ff655f9c46c1bb06af2dd539fde7
I20240502 07:58:47.258664 27645 tablet_bootstrap.cc:492] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:47.258738 27645 ts_tablet_manager.cc:1367] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:47.258829 27644 tablet_bootstrap.cc:654] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.259095 27645 raft_consensus.cc:348] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.259171 27645 raft_consensus.cc:374] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.259207 27645 raft_consensus.cc:729] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.259318 27645 consensus_queue.cc:260] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.259435 22868 tablet_copy_source_session.cc:205] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:47.259440 24983 tablet_service.cc:1467] Processing CreateTablet for tablet f25bbec445114d098d3af45d67c8a3b8 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=179ac83ba7bd4cd29d0cd00d9e40fdb5]), partition=
I20240502 07:58:47.259586 27645 ts_tablet_manager.cc:1398] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.259598 24983 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f25bbec445114d098d3af45d67c8a3b8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.259810 27136 raft_consensus.cc:482] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:47.259887 27136 raft_consensus.cc:504] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:47.260195 27136 leader_election.cc:288] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:47.260545 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "70ab3b4bafee41d883c87d30b63f756f" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:47.260635 24872 raft_consensus.cc:2454] T 70ab3b4bafee41d883c87d30b63f756f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:47.261078 22519 leader_election.cc:302] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:47.261243 27136 raft_consensus.cc:2781] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:47.261288 27136 raft_consensus.cc:482] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:47.261318 27136 raft_consensus.cc:3037] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.261955 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "70ab3b4bafee41d883c87d30b63f756f" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:47.262058 25266 raft_consensus.cc:2454] T 70ab3b4bafee41d883c87d30b63f756f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:47.262212 27644 tablet_bootstrap.cc:492] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:47.262279 27644 ts_tablet_manager.cc:1367] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.001s
I20240502 07:58:47.262362 27136 raft_consensus.cc:504] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:47.262696 27136 leader_election.cc:288] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:47.262724 27644 raft_consensus.cc:348] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.262817 27644 raft_consensus.cc:374] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.262866 27644 raft_consensus.cc:729] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.262964 27644 consensus_queue.cc:260] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.263238 27644 ts_tablet_manager.cc:1398] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:47.263342 27624 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ee06ff655f9c46c1bb06af2dd539fde7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.263731 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "70ab3b4bafee41d883c87d30b63f756f" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:47.263829 25266 raft_consensus.cc:3037] T 70ab3b4bafee41d883c87d30b63f756f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.264777 25266 raft_consensus.cc:2454] T 70ab3b4bafee41d883c87d30b63f756f P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:47.265064 22521 leader_election.cc:302] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7eced220cf1642f3873fc385b6191e69, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:47.265578 27643 tablet_bootstrap.cc:492] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:47.266184 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "70ab3b4bafee41d883c87d30b63f756f" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:47.266275 24872 raft_consensus.cc:3037] T 70ab3b4bafee41d883c87d30b63f756f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.266451 27643 tablet_bootstrap.cc:654] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.267012 24872 raft_consensus.cc:2454] T 70ab3b4bafee41d883c87d30b63f756f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:47.267611 27136 raft_consensus.cc:2781] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:47.267688 27136 raft_consensus.cc:686] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:47.267815 27136 consensus_queue.cc:237] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:47.268188 22586 consensus_queue.cc:237] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:47.268553 27624 tablet_copy_client.cc:783] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:47.268568 27643 tablet_bootstrap.cc:492] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:47.268640 27643 ts_tablet_manager.cc:1367] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240502 07:58:47.268759 27624 tablet_copy_client.cc:647] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:47.269057 27643 raft_consensus.cc:348] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.269140 27643 raft_consensus.cc:374] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.269181 27643 raft_consensus.cc:729] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.269285 27643 consensus_queue.cc:260] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.269498 27671 ts_tablet_manager.cc:902] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88: Initiating tablet copy from peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:47.269531 27643 ts_tablet_manager.cc:1398] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.269677 22845 raft_consensus.cc:1261] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:47.269738 27624 tablet_copy_client.cc:523] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:47.270498 27674 ts_tablet_manager.cc:902] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f: Initiating tablet copy from peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:47.270543 22440 catalog_manager.cc:5484] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:47.270781 27624 tablet_bootstrap.cc:492] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
W20240502 07:58:47.271062 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a5374313bbba4b10904e0ea18851c07d. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:47.271425 27674 tablet_copy_client.cc:308] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.136:34769
I20240502 07:58:47.271498 27136 consensus_queue.cc:1035] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:47.272091 25280 tablet_copy_service.cc:140] P 7eced220cf1642f3873fc385b6191e69: Received BeginTabletCopySession request for tablet 4e687b2e36574856a45752104ce83953 from peer 02a92e1c084b4308b9e855a4cc2c295f ({username='slave'} at 127.16.85.132:58659)
I20240502 07:58:47.272186 25280 tablet_copy_service.cc:161] P 7eced220cf1642f3873fc385b6191e69: Beginning new tablet copy session on tablet 4e687b2e36574856a45752104ce83953 from peer 02a92e1c084b4308b9e855a4cc2c295f at {username='slave'} at 127.16.85.132:58659: session id = 02a92e1c084b4308b9e855a4cc2c295f-4e687b2e36574856a45752104ce83953
I20240502 07:58:47.272889 22715 raft_consensus.cc:1261] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:47.273274 27136 consensus_queue.cc:1035] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:47.273463 25280 tablet_copy_source_session.cc:205] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:47.274916 27671 tablet_copy_client.cc:308] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.133:35459
I20240502 07:58:47.275014 27674 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4e687b2e36574856a45752104ce83953. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.275372 24892 tablet_copy_service.cc:140] P 6d02017318a2461796d431962fe6a537: Received BeginTabletCopySession request for tablet a071d3a1489e4d97ae6ceea4b0478459 from peer 7db4178eed174225bfb24f5f9445ce88 ({username='slave'} at 127.16.85.131:54881)
I20240502 07:58:47.275455 24892 tablet_copy_service.cc:161] P 6d02017318a2461796d431962fe6a537: Beginning new tablet copy session on tablet a071d3a1489e4d97ae6ceea4b0478459 from peer 7db4178eed174225bfb24f5f9445ce88 at {username='slave'} at 127.16.85.131:54881: session id = 7db4178eed174225bfb24f5f9445ce88-a071d3a1489e4d97ae6ceea4b0478459
I20240502 07:58:47.276566 24892 tablet_copy_source_session.cc:205] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:47.277523 27671 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a071d3a1489e4d97ae6ceea4b0478459. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.277524 27674 tablet_copy_client.cc:783] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:47.277681 27674 tablet_copy_client.cc:647] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:47.279459 27674 tablet_copy_client.cc:523] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:47.279824 27671 tablet_copy_client.cc:783] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:47.280001 27671 tablet_copy_client.cc:647] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:47.280956 27674 tablet_bootstrap.cc:492] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:47.280948 25987 raft_consensus.cc:2932] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:47.281792 27671 tablet_copy_client.cc:523] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:47.282997 27671 tablet_bootstrap.cc:492] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:47.283672 22440 catalog_manager.cc:5484] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:47.283771 22715 raft_consensus.cc:2932] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:47.287515 27671 tablet_bootstrap.cc:492] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:47.287729 27671 tablet_bootstrap.cc:492] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap complete.
I20240502 07:58:47.287850 27671 ts_tablet_manager.cc:1367] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240502 07:58:47.288224 27671 raft_consensus.cc:348] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 07:58:47.288317 27671 raft_consensus.cc:729] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: LEARNER
I20240502 07:58:47.288405 27671 consensus_queue.cc:260] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 07:58:47.290316 27671 ts_tablet_manager.cc:1398] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:47.291117 24892 tablet_copy_service.cc:342] P 6d02017318a2461796d431962fe6a537: Request end of tablet copy session 7db4178eed174225bfb24f5f9445ce88-a071d3a1489e4d97ae6ceea4b0478459 received from {username='slave'} at 127.16.85.131:54881
I20240502 07:58:47.291193 24892 tablet_copy_service.cc:433] P 6d02017318a2461796d431962fe6a537: ending tablet copy session 7db4178eed174225bfb24f5f9445ce88-a071d3a1489e4d97ae6ceea4b0478459 on tablet a071d3a1489e4d97ae6ceea4b0478459 with peer 7db4178eed174225bfb24f5f9445ce88
I20240502 07:58:47.293856 22845 raft_consensus.cc:2932] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:47.294885 27624 tablet_bootstrap.cc:492] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:47.295136 27624 tablet_bootstrap.cc:492] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537: Bootstrap complete.
I20240502 07:58:47.295291 27624 ts_tablet_manager.cc:1367] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.025s user 0.021s sys 0.000s
I20240502 07:58:47.295668 27624 raft_consensus.cc:348] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:47.295773 27624 raft_consensus.cc:729] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: LEARNER
I20240502 07:58:47.295868 27624 consensus_queue.cc:260] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:47.296090 27674 tablet_bootstrap.cc:492] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:47.296131 27624 ts_tablet_manager.cc:1398] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:47.296315 27674 tablet_bootstrap.cc:492] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap complete.
I20240502 07:58:47.296434 27674 ts_tablet_manager.cc:1367] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.016s user 0.004s sys 0.000s
I20240502 07:58:47.296541 22868 tablet_copy_service.cc:342] P 7db4178eed174225bfb24f5f9445ce88: Request end of tablet copy session 6d02017318a2461796d431962fe6a537-ee06ff655f9c46c1bb06af2dd539fde7 received from {username='slave'} at 127.16.85.133:51347
I20240502 07:58:47.296588 22868 tablet_copy_service.cc:433] P 7db4178eed174225bfb24f5f9445ce88: ending tablet copy session 6d02017318a2461796d431962fe6a537-ee06ff655f9c46c1bb06af2dd539fde7 on tablet ee06ff655f9c46c1bb06af2dd539fde7 with peer 6d02017318a2461796d431962fe6a537
I20240502 07:58:47.296824 27674 raft_consensus.cc:348] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 07:58:47.298199 27674 raft_consensus.cc:729] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: LEARNER
I20240502 07:58:47.298311 27674 consensus_queue.cc:260] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 07:58:47.298660 27674 ts_tablet_manager.cc:1398] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:47.299180 25280 tablet_copy_service.cc:342] P 7eced220cf1642f3873fc385b6191e69: Request end of tablet copy session 02a92e1c084b4308b9e855a4cc2c295f-4e687b2e36574856a45752104ce83953 received from {username='slave'} at 127.16.85.132:58659
I20240502 07:58:47.299264 25280 tablet_copy_service.cc:433] P 7eced220cf1642f3873fc385b6191e69: ending tablet copy session 02a92e1c084b4308b9e855a4cc2c295f-4e687b2e36574856a45752104ce83953 on tablet 4e687b2e36574856a45752104ce83953 with peer 02a92e1c084b4308b9e855a4cc2c295f
I20240502 07:58:47.302116 22715 consensus_queue.cc:237] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:47.303493 22845 raft_consensus.cc:1261] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:47.303629 22979 raft_consensus.cc:1261] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 07:58:47.303651 22653 consensus_peers.cc:485] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): Couldn't send request to peer 4654231dfe994c1a9d2e37d9c3d75699. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 55fb5c7475b14e0eaced5b8de1792641. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:47.303965 27582 consensus_queue.cc:1035] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:47.304172 27303 consensus_queue.cc:1035] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:47.305655 26995 raft_consensus.cc:2932] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:47.306048 22845 raft_consensus.cc:2932] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:47.306048 22979 raft_consensus.cc:2932] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:47.308239 22440 catalog_manager.cc:5484] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index -1 to 4, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:47.316358 26696 raft_consensus.cc:482] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:47.316449 26696 raft_consensus.cc:504] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.316834 26696 leader_election.cc:288] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:47.317101 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f25bbec445114d098d3af45d67c8a3b8" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:47.317183 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f25bbec445114d098d3af45d67c8a3b8" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:47.317232 24872 raft_consensus.cc:2454] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:47.317293 25134 raft_consensus.cc:2454] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:47.317528 24939 leader_election.cc:302] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:47.317782 26696 raft_consensus.cc:2781] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:47.317828 26696 raft_consensus.cc:482] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:47.317865 26696 raft_consensus.cc:3037] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.318756 26696 raft_consensus.cc:504] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.319000 26696 leader_election.cc:288] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:47.319306 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f25bbec445114d098d3af45d67c8a3b8" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:47.319409 25134 raft_consensus.cc:3037] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.319530 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f25bbec445114d098d3af45d67c8a3b8" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:47.319643 24872 raft_consensus.cc:3037] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.320286 25134 raft_consensus.cc:2454] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:47.320495 24872 raft_consensus.cc:2454] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:47.320811 24939 leader_election.cc:302] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:47.320953 26696 raft_consensus.cc:2781] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:47.321048 26696 raft_consensus.cc:686] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Becoming Leader. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Running, Role: LEADER
I20240502 07:58:47.321180 26696 consensus_queue.cc:237] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.322602 22442 catalog_manager.cc:5484] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: term changed from 0 to 1, leader changed from <none> to 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134). New cstate: current_term: 1 leader_uuid: "74f9aa91c2c448f980b512fe2e8d672e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:47.349959 27599 ts_tablet_manager.cc:902] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:47.350360 27599 tablet_copy_client.cc:308] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:47.350888 22999 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet cfff02438656430686be0f03f5c0b8ab from peer 4654231dfe994c1a9d2e37d9c3d75699 ({username='slave'} at 127.16.85.135:41731)
I20240502 07:58:47.350973 22999 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet cfff02438656430686be0f03f5c0b8ab from peer 4654231dfe994c1a9d2e37d9c3d75699 at {username='slave'} at 127.16.85.135:41731: session id = 4654231dfe994c1a9d2e37d9c3d75699-cfff02438656430686be0f03f5c0b8ab
I20240502 07:58:47.351269 27747 ts_tablet_manager.cc:902] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85: Initiating tablet copy from peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:47.351598 22999 tablet_copy_source_session.cc:205] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:47.351886 27747 tablet_copy_client.cc:308] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.131:44055
I20240502 07:58:47.352349 22868 tablet_copy_service.cc:140] P 7db4178eed174225bfb24f5f9445ce88: Received BeginTabletCopySession request for tablet 66ac69e448574c7ea5101014e0dd425d from peer 8bea0fe8d8044a2685f92f7a10c9cf85 ({username='slave'} at 127.16.85.129:51639)
I20240502 07:58:47.352407 27599 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cfff02438656430686be0f03f5c0b8ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.352440 22868 tablet_copy_service.cc:161] P 7db4178eed174225bfb24f5f9445ce88: Beginning new tablet copy session on tablet 66ac69e448574c7ea5101014e0dd425d from peer 8bea0fe8d8044a2685f92f7a10c9cf85 at {username='slave'} at 127.16.85.129:51639: session id = 8bea0fe8d8044a2685f92f7a10c9cf85-66ac69e448574c7ea5101014e0dd425d
I20240502 07:58:47.354112 22868 tablet_copy_source_session.cc:205] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:47.354739 27599 tablet_copy_client.cc:783] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:47.354871 27599 tablet_copy_client.cc:647] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:47.355262 27747 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 66ac69e448574c7ea5101014e0dd425d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.355939 27599 tablet_copy_client.cc:523] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:47.356987 27599 tablet_bootstrap.cc:492] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:47.358209 27747 tablet_copy_client.cc:783] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:47.358402 27747 tablet_copy_client.cc:647] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:47.360987 27747 tablet_copy_client.cc:523] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:47.362303 27747 tablet_bootstrap.cc:492] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:47.366132 27747 tablet_bootstrap.cc:492] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:47.366369 27747 tablet_bootstrap.cc:492] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap complete.
I20240502 07:58:47.366493 27747 ts_tablet_manager.cc:1367] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:47.366847 27747 raft_consensus.cc:348] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } }
I20240502 07:58:47.366950 27747 raft_consensus.cc:729] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: LEARNER
I20240502 07:58:47.367056 27747 consensus_queue.cc:260] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } }
I20240502 07:58:47.367332 27747 ts_tablet_manager.cc:1398] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 07:58:47.367893 22868 tablet_copy_service.cc:342] P 7db4178eed174225bfb24f5f9445ce88: Request end of tablet copy session 8bea0fe8d8044a2685f92f7a10c9cf85-66ac69e448574c7ea5101014e0dd425d received from {username='slave'} at 127.16.85.129:51639
I20240502 07:58:47.367960 22868 tablet_copy_service.cc:433] P 7db4178eed174225bfb24f5f9445ce88: ending tablet copy session 8bea0fe8d8044a2685f92f7a10c9cf85-66ac69e448574c7ea5101014e0dd425d on tablet 66ac69e448574c7ea5101014e0dd425d with peer 8bea0fe8d8044a2685f92f7a10c9cf85
I20240502 07:58:47.375885 27599 tablet_bootstrap.cc:492] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:47.376087 27599 tablet_bootstrap.cc:492] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap complete.
I20240502 07:58:47.376224 27599 ts_tablet_manager.cc:1367] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.019s user 0.011s sys 0.004s
I20240502 07:58:47.376600 27599 raft_consensus.cc:348] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:47.376765 27599 raft_consensus.cc:729] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: LEARNER
I20240502 07:58:47.376863 27599 consensus_queue.cc:260] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:47.377108 27599 ts_tablet_manager.cc:1398] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.004s sys 0.000s
I20240502 07:58:47.377499 22999 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-cfff02438656430686be0f03f5c0b8ab received from {username='slave'} at 127.16.85.135:41731
I20240502 07:58:47.377559 22999 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-cfff02438656430686be0f03f5c0b8ab on tablet cfff02438656430686be0f03f5c0b8ab with peer 4654231dfe994c1a9d2e37d9c3d75699
I20240502 07:58:47.405491 27599 ts_tablet_manager.cc:902] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699: Initiating tablet copy from peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:47.405908 27599 tablet_copy_client.cc:308] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.131:44055
I20240502 07:58:47.406397 22868 tablet_copy_service.cc:140] P 7db4178eed174225bfb24f5f9445ce88: Received BeginTabletCopySession request for tablet fcbab4ec41a74ff8b0c564ac7a0b044e from peer 4654231dfe994c1a9d2e37d9c3d75699 ({username='slave'} at 127.16.85.135:47281)
I20240502 07:58:47.406479 22868 tablet_copy_service.cc:161] P 7db4178eed174225bfb24f5f9445ce88: Beginning new tablet copy session on tablet fcbab4ec41a74ff8b0c564ac7a0b044e from peer 4654231dfe994c1a9d2e37d9c3d75699 at {username='slave'} at 127.16.85.135:47281: session id = 4654231dfe994c1a9d2e37d9c3d75699-fcbab4ec41a74ff8b0c564ac7a0b044e
I20240502 07:58:47.407300 22868 tablet_copy_source_session.cc:205] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:47.408025 27599 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fcbab4ec41a74ff8b0c564ac7a0b044e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.409770 27599 tablet_copy_client.cc:783] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:47.409904 27599 tablet_copy_client.cc:647] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:47.416839 27599 tablet_copy_client.cc:523] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:47.417872 27599 tablet_bootstrap.cc:492] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:47.418325 26284 raft_consensus.cc:1050] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88: attempting to promote NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e to VOTER
I20240502 07:58:47.418756 26284 consensus_queue.cc:237] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:47.420151 25003 raft_consensus.cc:1261] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:47.420267 22715 raft_consensus.cc:1261] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:47.420454 25532 consensus_queue.cc:1035] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:47.420925 22586 raft_consensus.cc:1261] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:47.421487 26284 consensus_queue.cc:1035] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:47.421617 25532 consensus_queue.cc:1035] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:47.425329 25532 raft_consensus.cc:2932] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:47.426960 22716 raft_consensus.cc:2932] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:47.429148 22436 catalog_manager.cc:5484] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:47.429572 25003 raft_consensus.cc:2932] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:47.431644 22586 raft_consensus.cc:2932] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:47.434542 22845 consensus_queue.cc:237] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:47.435679 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:47.435869 22716 raft_consensus.cc:1261] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:47.436844 26079 consensus_queue.cc:1035] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:47.437975 25003 raft_consensus.cc:1261] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:47.439035 25134 raft_consensus.cc:1261] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:47.439208 24872 raft_consensus.cc:1261] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:47.439321 26036 raft_consensus.cc:2932] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:47.439416 26696 consensus_queue.cc:1035] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:47.439985 26696 consensus_queue.cc:1035] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:47.440728 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6dbf36c0f044466ab13a53bfe0d6cb53 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:47.441033 22716 raft_consensus.cc:2932] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:47.442994 26284 consensus_queue.cc:1035] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:47.443305 22444 catalog_manager.cc:5484] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:47.443840 25003 raft_consensus.cc:2932] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:47.444509 22440 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet f25bbec445114d098d3af45d67c8a3b8 (table rebalancer_dynamic_table [id=179ac83ba7bd4cd29d0cd00d9e40fdb5]). Expected version 1 got 0
I20240502 07:58:47.445863 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 6dbf36c0f044466ab13a53bfe0d6cb53 with delete_type TABLET_DATA_TOMBSTONED (TS 8bea0fe8d8044a2685f92f7a10c9cf85 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:47.446033 25468 tablet_replica.cc:330] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:47.446166 25468 raft_consensus.cc:2229] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:47.446305 25468 raft_consensus.cc:2258] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.447295 22436 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet f25bbec445114d098d3af45d67c8a3b8 (table rebalancer_dynamic_table [id=179ac83ba7bd4cd29d0cd00d9e40fdb5]). Expected version 1 got 0
I20240502 07:58:47.447537 25468 ts_tablet_manager.cc:1875] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:47.449040 27481 tablet.cc:1722] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:47.449268 27481 tablet.cc:1620] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e: MemRowSet was empty: no flush needed.
I20240502 07:58:47.449556 25468 ts_tablet_manager.cc:1888] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:47.449625 25468 log.cc:1196] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/6dbf36c0f044466ab13a53bfe0d6cb53
I20240502 07:58:47.449880 27482 tablet.cc:1722] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:47.450155 27482 tablet.cc:1620] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:47.450989 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 6dbf36c0f044466ab13a53bfe0d6cb53 (table rebalance_test_table_7 [id=b66f182129cf49c0a53671ec66e4a6e2]) successfully deleted
I20240502 07:58:47.451519 25135 raft_consensus.cc:1203] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:47.454365 27815 ts_tablet_manager.cc:902] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3: Initiating tablet copy from peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:47.455046 27483 tablet.cc:1722] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:47.455130 27815 tablet_copy_client.cc:308] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.136:34769
I20240502 07:58:47.455312 27483 tablet.cc:1620] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:47.455855 25280 tablet_copy_service.cc:140] P 7eced220cf1642f3873fc385b6191e69: Received BeginTabletCopySession request for tablet cfcaa9ee1fbe48ae91796b184ecaf2a3 from peer 725969d490a04fdfb3ba890c3f5096e3 ({username='slave'} at 127.16.85.130:41473)
I20240502 07:58:47.455950 25280 tablet_copy_service.cc:161] P 7eced220cf1642f3873fc385b6191e69: Beginning new tablet copy session on tablet cfcaa9ee1fbe48ae91796b184ecaf2a3 from peer 725969d490a04fdfb3ba890c3f5096e3 at {username='slave'} at 127.16.85.130:41473: session id = 725969d490a04fdfb3ba890c3f5096e3-cfcaa9ee1fbe48ae91796b184ecaf2a3
I20240502 07:58:47.456813 22436 catalog_manager.cc:5576] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857) does not have the latest schema for tablet f25bbec445114d098d3af45d67c8a3b8 (table rebalancer_dynamic_table [id=179ac83ba7bd4cd29d0cd00d9e40fdb5]). Expected version 1 got 0
I20240502 07:58:47.456866 25280 tablet_copy_source_session.cc:205] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:47.457013 22442 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=179ac83ba7bd4cd29d0cd00d9e40fdb5] alter complete (version 1)
I20240502 07:58:47.458600 27816 ts_tablet_manager.cc:902] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:47.459031 27815 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cfcaa9ee1fbe48ae91796b184ecaf2a3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.459126 27816 tablet_copy_client.cc:308] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33387
I20240502 07:58:47.459518 22737 tablet_copy_service.cc:140] P 725969d490a04fdfb3ba890c3f5096e3: Received BeginTabletCopySession request for tablet cceaa8887081401dbc9154d265004c7b from peer 74f9aa91c2c448f980b512fe2e8d672e ({username='slave'} at 127.16.85.134:53317)
I20240502 07:58:47.459604 22737 tablet_copy_service.cc:161] P 725969d490a04fdfb3ba890c3f5096e3: Beginning new tablet copy session on tablet cceaa8887081401dbc9154d265004c7b from peer 74f9aa91c2c448f980b512fe2e8d672e at {username='slave'} at 127.16.85.134:53317: session id = 74f9aa91c2c448f980b512fe2e8d672e-cceaa8887081401dbc9154d265004c7b
I20240502 07:58:47.460470 22737 tablet_copy_source_session.cc:205] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:47.462309 27816 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cceaa8887081401dbc9154d265004c7b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.462838 27815 tablet_copy_client.cc:783] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:47.463043 27815 tablet_copy_client.cc:647] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:47.464083 27599 tablet_bootstrap.cc:492] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:47.464320 27599 tablet_bootstrap.cc:492] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap complete.
I20240502 07:58:47.464496 27816 tablet_copy_client.cc:783] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:47.464514 27599 ts_tablet_manager.cc:1367] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.047s user 0.021s sys 0.001s
I20240502 07:58:47.464622 27816 tablet_copy_client.cc:647] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:47.464929 27599 raft_consensus.cc:348] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:47.465039 27599 raft_consensus.cc:729] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: LEARNER
I20240502 07:58:47.465143 27599 consensus_queue.cc:260] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:47.465411 27599 ts_tablet_manager.cc:1398] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 07:58:47.465778 27816 tablet_copy_client.cc:523] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:47.466153 22868 tablet_copy_service.cc:342] P 7db4178eed174225bfb24f5f9445ce88: Request end of tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-fcbab4ec41a74ff8b0c564ac7a0b044e received from {username='slave'} at 127.16.85.135:47281
I20240502 07:58:47.466229 22868 tablet_copy_service.cc:433] P 7db4178eed174225bfb24f5f9445ce88: ending tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-fcbab4ec41a74ff8b0c564ac7a0b044e on tablet fcbab4ec41a74ff8b0c564ac7a0b044e with peer 4654231dfe994c1a9d2e37d9c3d75699
I20240502 07:58:47.467193 27816 tablet_bootstrap.cc:492] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:47.471757 27815 tablet_copy_client.cc:523] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:47.472223 24872 consensus_queue.cc:237] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 07:58:47.473502 27815 tablet_bootstrap.cc:492] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:47.473660 25266 raft_consensus.cc:1261] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:47.474088 25134 raft_consensus.cc:1261] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:47.474061 27338 consensus_queue.cc:1035] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
W20240502 07:58:47.474844 24808 consensus_peers.cc:485] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 -> Peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): Couldn't send request to peer 7db4178eed174225bfb24f5f9445ce88. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3bb108b6ab55461fb9fabf431d953ef2. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:47.475526 27338 consensus_queue.cc:1035] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:47.477027 27338 raft_consensus.cc:2932] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:47.477643 27815 tablet_bootstrap.cc:492] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:47.477949 27815 tablet_bootstrap.cc:492] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap complete.
I20240502 07:58:47.478093 27815 ts_tablet_manager.cc:1367] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.004s
I20240502 07:58:47.478513 27815 raft_consensus.cc:348] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 07:58:47.478672 27815 raft_consensus.cc:729] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: LEARNER
I20240502 07:58:47.478793 27815 consensus_queue.cc:260] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 07:58:47.479208 27815 ts_tablet_manager.cc:1398] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:47.479197 25266 raft_consensus.cc:2932] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:47.480106 25134 raft_consensus.cc:2932] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:47.481319 25280 tablet_copy_service.cc:342] P 7eced220cf1642f3873fc385b6191e69: Request end of tablet copy session 725969d490a04fdfb3ba890c3f5096e3-cfcaa9ee1fbe48ae91796b184ecaf2a3 received from {username='slave'} at 127.16.85.130:41473
I20240502 07:58:47.481400 25280 tablet_copy_service.cc:433] P 7eced220cf1642f3873fc385b6191e69: ending tablet copy session 725969d490a04fdfb3ba890c3f5096e3-cfcaa9ee1fbe48ae91796b184ecaf2a3 on tablet cfcaa9ee1fbe48ae91796b184ecaf2a3 with peer 725969d490a04fdfb3ba890c3f5096e3
I20240502 07:58:47.481998 27828 ts_tablet_manager.cc:902] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:47.482280 22442 catalog_manager.cc:5484] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:47.482532 27828 tablet_copy_client.cc:308] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33387
I20240502 07:58:47.483023 22737 tablet_copy_service.cc:140] P 725969d490a04fdfb3ba890c3f5096e3: Received BeginTabletCopySession request for tablet c3a7b2e52ba641e993cf1d5dc9724473 from peer 74f9aa91c2c448f980b512fe2e8d672e ({username='slave'} at 127.16.85.134:53317)
I20240502 07:58:47.483117 22737 tablet_copy_service.cc:161] P 725969d490a04fdfb3ba890c3f5096e3: Beginning new tablet copy session on tablet c3a7b2e52ba641e993cf1d5dc9724473 from peer 74f9aa91c2c448f980b512fe2e8d672e at {username='slave'} at 127.16.85.134:53317: session id = 74f9aa91c2c448f980b512fe2e8d672e-c3a7b2e52ba641e993cf1d5dc9724473
I20240502 07:58:47.483956 22737 tablet_copy_source_session.cc:205] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:47.485595 27828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c3a7b2e52ba641e993cf1d5dc9724473. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.487306 27828 tablet_copy_client.cc:783] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:47.487444 27828 tablet_copy_client.cc:647] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:47.488858 27828 tablet_copy_client.cc:523] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:47.490046 27828 tablet_bootstrap.cc:492] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:47.493587 27828 tablet_bootstrap.cc:492] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:47.493922 27828 tablet_bootstrap.cc:492] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap complete.
I20240502 07:58:47.494077 27828 ts_tablet_manager.cc:1367] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 07:58:47.494550 27828 raft_consensus.cc:348] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:47.494781 27828 raft_consensus.cc:729] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: LEARNER
I20240502 07:58:47.494884 27828 consensus_queue.cc:260] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:47.495028 27816 tablet_bootstrap.cc:492] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:47.495131 27828 ts_tablet_manager.cc:1398] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.495198 27816 tablet_bootstrap.cc:492] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap complete.
I20240502 07:58:47.495318 27816 ts_tablet_manager.cc:1367] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.028s user 0.024s sys 0.000s
I20240502 07:58:47.495671 27816 raft_consensus.cc:348] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:47.495733 22737 tablet_copy_service.cc:342] P 725969d490a04fdfb3ba890c3f5096e3: Request end of tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-c3a7b2e52ba641e993cf1d5dc9724473 received from {username='slave'} at 127.16.85.134:53317
I20240502 07:58:47.495771 27816 raft_consensus.cc:729] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: LEARNER
I20240502 07:58:47.495797 22737 tablet_copy_service.cc:433] P 725969d490a04fdfb3ba890c3f5096e3: ending tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-c3a7b2e52ba641e993cf1d5dc9724473 on tablet c3a7b2e52ba641e993cf1d5dc9724473 with peer 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:47.495998 27816 consensus_queue.cc:260] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:47.496266 27816 ts_tablet_manager.cc:1398] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.003s sys 0.000s
I20240502 07:58:47.496616 22586 consensus_queue.cc:237] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:47.497839 22737 tablet_copy_service.cc:342] P 725969d490a04fdfb3ba890c3f5096e3: Request end of tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-cceaa8887081401dbc9154d265004c7b received from {username='slave'} at 127.16.85.134:53317
I20240502 07:58:47.497901 22737 tablet_copy_service.cc:433] P 725969d490a04fdfb3ba890c3f5096e3: ending tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-cceaa8887081401dbc9154d265004c7b on tablet cceaa8887081401dbc9154d265004c7b with peer 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:47.498639 22716 raft_consensus.cc:1261] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 07:58:47.498756 25266 raft_consensus.cc:1261] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 07:58:47.499060 27023 consensus_queue.cc:1035] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:47.499204 24714 consensus_queue.cc:1035] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240502 07:58:47.499379 22519 consensus_peers.cc:485] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): Couldn't send request to peer 6d02017318a2461796d431962fe6a537. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c53e8b3bd774466dbafe4994472975bc. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:47.500824 24714 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:47.501331 22716 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:47.501348 25266 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:47.503512 22436 catalog_manager.cc:5484] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 6 to 7, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:47.549237 27671 ts_tablet_manager.cc:902] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88: Initiating tablet copy from peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:47.549611 27671 tablet_copy_client.cc:308] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:37451
I20240502 07:58:47.549983 22606 tablet_copy_service.cc:140] P 8bea0fe8d8044a2685f92f7a10c9cf85: Received BeginTabletCopySession request for tablet 415879c2eee74932b9ac45282f6dd0fb from peer 7db4178eed174225bfb24f5f9445ce88 ({username='slave'} at 127.16.85.131:36129)
I20240502 07:58:47.550074 22606 tablet_copy_service.cc:161] P 8bea0fe8d8044a2685f92f7a10c9cf85: Beginning new tablet copy session on tablet 415879c2eee74932b9ac45282f6dd0fb from peer 7db4178eed174225bfb24f5f9445ce88 at {username='slave'} at 127.16.85.131:36129: session id = 7db4178eed174225bfb24f5f9445ce88-415879c2eee74932b9ac45282f6dd0fb
I20240502 07:58:47.550714 22606 tablet_copy_source_session.cc:205] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:47.551584 27671 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 415879c2eee74932b9ac45282f6dd0fb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.552986 27891 ts_tablet_manager.cc:902] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:47.553532 27891 tablet_copy_client.cc:308] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33387
I20240502 07:58:47.553711 27671 tablet_copy_client.cc:783] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:47.553870 27671 tablet_copy_client.cc:647] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:47.554014 22737 tablet_copy_service.cc:140] P 725969d490a04fdfb3ba890c3f5096e3: Received BeginTabletCopySession request for tablet e25ba5cc77334a3595a4575fa98476f0 from peer 7eced220cf1642f3873fc385b6191e69 ({username='slave'} at 127.16.85.136:54683)
I20240502 07:58:47.554111 22737 tablet_copy_service.cc:161] P 725969d490a04fdfb3ba890c3f5096e3: Beginning new tablet copy session on tablet e25ba5cc77334a3595a4575fa98476f0 from peer 7eced220cf1642f3873fc385b6191e69 at {username='slave'} at 127.16.85.136:54683: session id = 7eced220cf1642f3873fc385b6191e69-e25ba5cc77334a3595a4575fa98476f0
I20240502 07:58:47.554922 22737 tablet_copy_source_session.cc:205] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:47.556106 27671 tablet_copy_client.cc:523] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:47.557195 27671 tablet_bootstrap.cc:492] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:47.557332 27891 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e25ba5cc77334a3595a4575fa98476f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.559406 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:47.560235 27671 tablet_bootstrap.cc:492] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:47.560552 27671 tablet_bootstrap.cc:492] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88: Bootstrap complete.
I20240502 07:58:47.560798 27671 ts_tablet_manager.cc:1367] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.004s user 0.004s sys 0.000s
I20240502 07:58:47.561228 27671 raft_consensus.cc:348] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 07:58:47.561384 27671 raft_consensus.cc:729] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: LEARNER
I20240502 07:58:47.561535 27671 consensus_queue.cc:260] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 07:58:47.562192 27671 ts_tablet_manager.cc:1398] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.562608 22606 tablet_copy_service.cc:342] P 8bea0fe8d8044a2685f92f7a10c9cf85: Request end of tablet copy session 7db4178eed174225bfb24f5f9445ce88-415879c2eee74932b9ac45282f6dd0fb received from {username='slave'} at 127.16.85.131:36129
I20240502 07:58:47.562674 22606 tablet_copy_service.cc:433] P 8bea0fe8d8044a2685f92f7a10c9cf85: ending tablet copy session 7db4178eed174225bfb24f5f9445ce88-415879c2eee74932b9ac45282f6dd0fb on tablet 415879c2eee74932b9ac45282f6dd0fb with peer 7db4178eed174225bfb24f5f9445ce88
I20240502 07:58:47.565080 27891 tablet_copy_client.cc:783] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:47.565198 27481 tablet.cc:1722] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:47.565258 27891 tablet_copy_client.cc:647] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:47.565476 27481 tablet.cc:1620] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e: MemRowSet was empty: no flush needed.
I20240502 07:58:47.566613 27891 tablet_copy_client.cc:523] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:47.566744 27482 tablet.cc:1722] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:47.567006 27482 tablet.cc:1620] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:47.567811 27891 tablet_bootstrap.cc:492] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:47.569787 27483 tablet.cc:1722] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:47.570065 27483 tablet.cc:1620] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:47.571386 22442 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=179ac83ba7bd4cd29d0cd00d9e40fdb5] alter complete (version 2)
I20240502 07:58:47.598780 27891 tablet_bootstrap.cc:492] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:47.599018 27891 tablet_bootstrap.cc:492] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69: Bootstrap complete.
I20240502 07:58:47.599171 27891 ts_tablet_manager.cc:1367] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.031s user 0.022s sys 0.000s
I20240502 07:58:47.599526 27891 raft_consensus.cc:348] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:47.599628 27891 raft_consensus.cc:729] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: LEARNER
I20240502 07:58:47.599704 27891 consensus_queue.cc:260] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:47.599973 27891 ts_tablet_manager.cc:1398] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 07:58:47.600387 22737 tablet_copy_service.cc:342] P 725969d490a04fdfb3ba890c3f5096e3: Request end of tablet copy session 7eced220cf1642f3873fc385b6191e69-e25ba5cc77334a3595a4575fa98476f0 received from {username='slave'} at 127.16.85.136:54683
I20240502 07:58:47.600466 22737 tablet_copy_service.cc:433] P 725969d490a04fdfb3ba890c3f5096e3: ending tablet copy session 7eced220cf1642f3873fc385b6191e69-e25ba5cc77334a3595a4575fa98476f0 on tablet e25ba5cc77334a3595a4575fa98476f0 with peer 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:47.633605 24872 raft_consensus.cc:1203] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:47.657196 24872 raft_consensus.cc:3037] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.658557 27488 consensus_queue.cc:1035] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:47.661953 27488 consensus_queue.cc:1035] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:47.665077 27136 consensus_queue.cc:1035] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:47.668516 27023 consensus_queue.cc:1035] T 70ab3b4bafee41d883c87d30b63f756f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:47.671808 27110 consensus_queue.cc:1035] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:47.676081 27339 consensus_queue.cc:1035] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:47.678275 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:47.684118 27481 tablet.cc:1722] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:47.684830 27482 tablet.cc:1722] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:47.686352 27483 tablet.cc:1722] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:47.690387 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=179ac83ba7bd4cd29d0cd00d9e40fdb5] alter complete (version 3)
I20240502 07:58:47.724818 22979 raft_consensus.cc:1203] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 07:58:47.756850 27816 ts_tablet_manager.cc:902] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e: Initiating tablet copy from peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:47.757303 27816 tablet_copy_client.cc:308] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:37451
I20240502 07:58:47.758710 22606 tablet_copy_service.cc:140] P 8bea0fe8d8044a2685f92f7a10c9cf85: Received BeginTabletCopySession request for tablet a5374313bbba4b10904e0ea18851c07d from peer 74f9aa91c2c448f980b512fe2e8d672e ({username='slave'} at 127.16.85.134:42087)
I20240502 07:58:47.758798 22606 tablet_copy_service.cc:161] P 8bea0fe8d8044a2685f92f7a10c9cf85: Beginning new tablet copy session on tablet a5374313bbba4b10904e0ea18851c07d from peer 74f9aa91c2c448f980b512fe2e8d672e at {username='slave'} at 127.16.85.134:42087: session id = 74f9aa91c2c448f980b512fe2e8d672e-a5374313bbba4b10904e0ea18851c07d
I20240502 07:58:47.759490 22606 tablet_copy_source_session.cc:205] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:47.760318 27816 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a5374313bbba4b10904e0ea18851c07d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.762115 27816 tablet_copy_client.cc:783] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:47.762246 27816 tablet_copy_client.cc:647] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:47.763638 27816 tablet_copy_client.cc:523] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:47.764597 27816 tablet_bootstrap.cc:492] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:47.766844 25134 raft_consensus.cc:1203] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 07:58:47.781299 27816 tablet_bootstrap.cc:492] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:47.781517 27816 tablet_bootstrap.cc:492] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap complete.
I20240502 07:58:47.781656 27816 ts_tablet_manager.cc:1367] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.017s user 0.017s sys 0.000s
I20240502 07:58:47.781980 27816 raft_consensus.cc:348] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:47.782068 27816 raft_consensus.cc:729] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: LEARNER
I20240502 07:58:47.782151 27816 consensus_queue.cc:260] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:47.782366 27816 ts_tablet_manager.cc:1398] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:47.782775 22606 tablet_copy_service.cc:342] P 8bea0fe8d8044a2685f92f7a10c9cf85: Request end of tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-a5374313bbba4b10904e0ea18851c07d received from {username='slave'} at 127.16.85.134:42087
I20240502 07:58:47.782852 22606 tablet_copy_service.cc:433] P 8bea0fe8d8044a2685f92f7a10c9cf85: ending tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-a5374313bbba4b10904e0ea18851c07d on tablet a5374313bbba4b10904e0ea18851c07d with peer 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:47.793684 22442 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:47.793809 22442 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:47.795604 22442 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet f25bbec445114d098d3af45d67c8a3b8
I20240502 07:58:47.796137 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet f25bbec445114d098d3af45d67c8a3b8 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:47 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:47.796161 25113 tablet_service.cc:1514] Processing DeleteTablet for tablet f25bbec445114d098d3af45d67c8a3b8 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:47 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:47.796424 28042 tablet_replica.cc:330] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:47.796422 28041 tablet_replica.cc:330] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:47.796497 24983 tablet_service.cc:1514] Processing DeleteTablet for tablet f25bbec445114d098d3af45d67c8a3b8 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:47 UTC) from {username='slave'} at 127.0.0.1:42878
I20240502 07:58:47.796561 28042 raft_consensus.cc:2229] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:47.796608 28041 raft_consensus.cc:2229] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:47.796689 28042 raft_consensus.cc:2258] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.796759 28041 raft_consensus.cc:2258] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.796777 28043 tablet_replica.cc:330] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e: stopping tablet replica
I20240502 07:58:47.796880 28043 raft_consensus.cc:2229] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:47.797083 28043 raft_consensus.cc:2258] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.797307 28042 ts_tablet_manager.cc:1875] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:47.797350 28041 ts_tablet_manager.cc:1875] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:47.797645 28043 ts_tablet_manager.cc:1875] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:47.798633 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_14"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:47.799192 28041 ts_tablet_manager.cc:1888] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:47.799255 28041 log.cc:1196] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/f25bbec445114d098d3af45d67c8a3b8
I20240502 07:58:47.799444 28041 ts_tablet_manager.cc:1909] T f25bbec445114d098d3af45d67c8a3b8 P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:47.799686 28042 ts_tablet_manager.cc:1888] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:47.799744 28042 log.cc:1196] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/f25bbec445114d098d3af45d67c8a3b8
I20240502 07:58:47.800048 28042 ts_tablet_manager.cc:1909] T f25bbec445114d098d3af45d67c8a3b8 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:47.800643 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet f25bbec445114d098d3af45d67c8a3b8 (table rebalancer_dynamic_table_new_name [id=179ac83ba7bd4cd29d0cd00d9e40fdb5]) successfully deleted
I20240502 07:58:47.801568 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet f25bbec445114d098d3af45d67c8a3b8 (table rebalancer_dynamic_table_new_name [id=179ac83ba7bd4cd29d0cd00d9e40fdb5]) successfully deleted
I20240502 07:58:47.802088 28043 ts_tablet_manager.cc:1888] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:47.802189 28043 log.cc:1196] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/wals/f25bbec445114d098d3af45d67c8a3b8
I20240502 07:58:47.802424 28043 ts_tablet_manager.cc:1909] T f25bbec445114d098d3af45d67c8a3b8 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting consensus metadata
I20240502 07:58:47.802933 22430 catalog_manager.cc:4830] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): tablet f25bbec445114d098d3af45d67c8a3b8 (table rebalancer_dynamic_table_new_name [id=179ac83ba7bd4cd29d0cd00d9e40fdb5]) successfully deleted
I20240502 07:58:47.804517 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 9f55942f203d455394d2f81f9ae91f0f (DEFAULT_TABLE table=rebalancer_extra_table_14 [id=0713b54eb3de4d46b87bb75569629365]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.804797 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f55942f203d455394d2f81f9ae91f0f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.805009 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 0506cd627d684ac784a5c738f87389af (DEFAULT_TABLE table=rebalancer_extra_table_14 [id=0713b54eb3de4d46b87bb75569629365]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.805240 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0506cd627d684ac784a5c738f87389af. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.805435 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 0506cd627d684ac784a5c738f87389af (DEFAULT_TABLE table=rebalancer_extra_table_14 [id=0713b54eb3de4d46b87bb75569629365]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.805675 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0506cd627d684ac784a5c738f87389af. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.806442 28044 tablet_bootstrap.cc:492] T 9f55942f203d455394d2f81f9ae91f0f P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:47.806955 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 9f55942f203d455394d2f81f9ae91f0f (DEFAULT_TABLE table=rebalancer_extra_table_14 [id=0713b54eb3de4d46b87bb75569629365]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.807135 24851 tablet_service.cc:1467] Processing CreateTablet for tablet 05c1913e62aa4be388888d6fa65c8523 (DEFAULT_TABLE table=rebalancer_extra_table_14 [id=0713b54eb3de4d46b87bb75569629365]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.807200 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f55942f203d455394d2f81f9ae91f0f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.807333 24851 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 05c1913e62aa4be388888d6fa65c8523. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.807739 28045 tablet_bootstrap.cc:492] T 0506cd627d684ac784a5c738f87389af P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:47.807976 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 0506cd627d684ac784a5c738f87389af (DEFAULT_TABLE table=rebalancer_extra_table_14 [id=0713b54eb3de4d46b87bb75569629365]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.808207 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0506cd627d684ac784a5c738f87389af. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.808723 28045 tablet_bootstrap.cc:654] T 0506cd627d684ac784a5c738f87389af P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.809384 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 9f55942f203d455394d2f81f9ae91f0f (DEFAULT_TABLE table=rebalancer_extra_table_14 [id=0713b54eb3de4d46b87bb75569629365]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.809576 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f55942f203d455394d2f81f9ae91f0f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.809574 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 05c1913e62aa4be388888d6fa65c8523 (DEFAULT_TABLE table=rebalancer_extra_table_14 [id=0713b54eb3de4d46b87bb75569629365]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.809650 28045 tablet_bootstrap.cc:492] T 0506cd627d684ac784a5c738f87389af P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:47.809716 28045 ts_tablet_manager.cc:1367] T 0506cd627d684ac784a5c738f87389af P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:47.809810 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 05c1913e62aa4be388888d6fa65c8523. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.810118 28045 raft_consensus.cc:348] T 0506cd627d684ac784a5c738f87389af P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:47.810233 28045 raft_consensus.cc:374] T 0506cd627d684ac784a5c738f87389af P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.810266 28045 raft_consensus.cc:729] T 0506cd627d684ac784a5c738f87389af P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.810366 28045 consensus_queue.cc:260] T 0506cd627d684ac784a5c738f87389af P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:47.810621 28045 ts_tablet_manager.cc:1398] T 0506cd627d684ac784a5c738f87389af P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.811617 28049 tablet_bootstrap.cc:492] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:47.811901 28050 tablet_bootstrap.cc:492] T 05c1913e62aa4be388888d6fa65c8523 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:47.812377 28049 tablet_bootstrap.cc:654] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.812763 22845 raft_consensus.cc:1203] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 07:58:47.812800 28050 tablet_bootstrap.cc:654] T 05c1913e62aa4be388888d6fa65c8523 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.813357 28049 tablet_bootstrap.cc:492] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:47.813445 28049 ts_tablet_manager.cc:1367] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:47.813832 28050 tablet_bootstrap.cc:492] T 05c1913e62aa4be388888d6fa65c8523 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:47.813923 28050 ts_tablet_manager.cc:1367] T 05c1913e62aa4be388888d6fa65c8523 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:47.813922 28049 raft_consensus.cc:348] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:47.814018 28049 raft_consensus.cc:374] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.814055 28049 raft_consensus.cc:729] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.814163 28049 consensus_queue.cc:260] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:47.814342 28050 raft_consensus.cc:348] T 05c1913e62aa4be388888d6fa65c8523 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.814438 28050 raft_consensus.cc:374] T 05c1913e62aa4be388888d6fa65c8523 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.814445 28049 ts_tablet_manager.cc:1398] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.814383 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 05c1913e62aa4be388888d6fa65c8523 (DEFAULT_TABLE table=rebalancer_extra_table_14 [id=0713b54eb3de4d46b87bb75569629365]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:47.814484 28050 raft_consensus.cc:729] T 05c1913e62aa4be388888d6fa65c8523 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.814510 28044 tablet_bootstrap.cc:654] T 9f55942f203d455394d2f81f9ae91f0f P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.814628 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 05c1913e62aa4be388888d6fa65c8523. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.814589 28050 consensus_queue.cc:260] T 05c1913e62aa4be388888d6fa65c8523 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.814875 28050 ts_tablet_manager.cc:1398] T 05c1913e62aa4be388888d6fa65c8523 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.815439 28044 tablet_bootstrap.cc:492] T 9f55942f203d455394d2f81f9ae91f0f P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:47.815515 28044 ts_tablet_manager.cc:1367] T 9f55942f203d455394d2f81f9ae91f0f P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.009s user 0.001s sys 0.000s
I20240502 07:58:47.815935 28044 raft_consensus.cc:348] T 9f55942f203d455394d2f81f9ae91f0f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.816037 28044 raft_consensus.cc:374] T 9f55942f203d455394d2f81f9ae91f0f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.816069 28044 raft_consensus.cc:729] T 9f55942f203d455394d2f81f9ae91f0f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.816181 28044 consensus_queue.cc:260] T 9f55942f203d455394d2f81f9ae91f0f P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.816433 28044 ts_tablet_manager.cc:1398] T 9f55942f203d455394d2f81f9ae91f0f P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.816602 28044 tablet_bootstrap.cc:492] T 0506cd627d684ac784a5c738f87389af P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:47.817425 28044 tablet_bootstrap.cc:654] T 0506cd627d684ac784a5c738f87389af P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.818519 25134 raft_consensus.cc:1203] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:47.821377 28047 tablet_bootstrap.cc:492] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:47.822322 28047 tablet_bootstrap.cc:654] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.822532 28046 tablet_bootstrap.cc:492] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:47.823887 28044 tablet_bootstrap.cc:492] T 0506cd627d684ac784a5c738f87389af P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:47.823983 28044 ts_tablet_manager.cc:1367] T 0506cd627d684ac784a5c738f87389af P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 07:58:47.824088 28046 tablet_bootstrap.cc:654] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.824497 28047 tablet_bootstrap.cc:492] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:47.824437 28044 raft_consensus.cc:348] T 0506cd627d684ac784a5c738f87389af P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:47.824546 28044 raft_consensus.cc:374] T 0506cd627d684ac784a5c738f87389af P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.824584 28044 raft_consensus.cc:729] T 0506cd627d684ac784a5c738f87389af P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.824635 28047 ts_tablet_manager.cc:1367] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:47.824784 28044 consensus_queue.cc:260] T 0506cd627d684ac784a5c738f87389af P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:47.825117 28047 raft_consensus.cc:348] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.825251 28047 raft_consensus.cc:374] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.825322 28047 raft_consensus.cc:729] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.825372 28046 tablet_bootstrap.cc:492] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:47.825508 28046 ts_tablet_manager.cc:1367] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:47.825489 28047 consensus_queue.cc:260] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.825654 28044 ts_tablet_manager.cc:1398] T 0506cd627d684ac784a5c738f87389af P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:47.825875 28044 tablet_bootstrap.cc:492] T 05c1913e62aa4be388888d6fa65c8523 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:47.825986 28046 raft_consensus.cc:348] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.826119 28046 raft_consensus.cc:374] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.826185 28046 raft_consensus.cc:729] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.826370 28046 consensus_queue.cc:260] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.827123 28046 ts_tablet_manager.cc:1398] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:47.827322 28046 tablet_bootstrap.cc:492] T 9f55942f203d455394d2f81f9ae91f0f P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:47.827323 28044 tablet_bootstrap.cc:654] T 05c1913e62aa4be388888d6fa65c8523 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.827500 28047 ts_tablet_manager.cc:1398] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.003s user 0.000s sys 0.001s
I20240502 07:58:47.828220 28046 tablet_bootstrap.cc:654] T 9f55942f203d455394d2f81f9ae91f0f P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.828248 28044 tablet_bootstrap.cc:492] T 05c1913e62aa4be388888d6fa65c8523 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:47.828316 28044 ts_tablet_manager.cc:1367] T 05c1913e62aa4be388888d6fa65c8523 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:47.828754 28044 raft_consensus.cc:348] T 05c1913e62aa4be388888d6fa65c8523 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.828851 28044 raft_consensus.cc:374] T 05c1913e62aa4be388888d6fa65c8523 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.828900 28044 raft_consensus.cc:729] T 05c1913e62aa4be388888d6fa65c8523 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.829047 28044 consensus_queue.cc:260] T 05c1913e62aa4be388888d6fa65c8523 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.829267 28046 tablet_bootstrap.cc:492] T 9f55942f203d455394d2f81f9ae91f0f P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:47.829290 28044 ts_tablet_manager.cc:1398] T 05c1913e62aa4be388888d6fa65c8523 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.829353 28046 ts_tablet_manager.cc:1367] T 9f55942f203d455394d2f81f9ae91f0f P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:47.829743 28046 raft_consensus.cc:348] T 9f55942f203d455394d2f81f9ae91f0f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.829859 28046 raft_consensus.cc:374] T 9f55942f203d455394d2f81f9ae91f0f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.829893 28046 raft_consensus.cc:729] T 9f55942f203d455394d2f81f9ae91f0f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.830003 28046 consensus_queue.cc:260] T 9f55942f203d455394d2f81f9ae91f0f P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.830237 28046 ts_tablet_manager.cc:1398] T 9f55942f203d455394d2f81f9ae91f0f P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.841645 25003 raft_consensus.cc:1203] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:47.843369 22586 raft_consensus.cc:1203] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 07:58:47.843962 27582 raft_consensus.cc:482] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:47.844053 27582 raft_consensus.cc:504] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:47.844528 27582 leader_election.cc:288] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:47.844949 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0506cd627d684ac784a5c738f87389af" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:47.844954 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0506cd627d684ac784a5c738f87389af" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:47.845064 22586 raft_consensus.cc:2454] T 0506cd627d684ac784a5c738f87389af P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:47.845089 25134 raft_consensus.cc:2454] T 0506cd627d684ac784a5c738f87389af P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:47.845551 22653 leader_election.cc:302] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:47.845702 27582 raft_consensus.cc:2781] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:47.845786 27582 raft_consensus.cc:482] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:47.845834 27582 raft_consensus.cc:3037] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.846623 27582 raft_consensus.cc:504] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:47.846933 27582 leader_election.cc:288] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:47.847229 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0506cd627d684ac784a5c738f87389af" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:47.847338 22586 raft_consensus.cc:3037] T 0506cd627d684ac784a5c738f87389af P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.847338 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0506cd627d684ac784a5c738f87389af" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:47.847455 25134 raft_consensus.cc:3037] T 0506cd627d684ac784a5c738f87389af P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.848207 22586 raft_consensus.cc:2454] T 0506cd627d684ac784a5c738f87389af P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:47.848348 25134 raft_consensus.cc:2454] T 0506cd627d684ac784a5c738f87389af P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:47.848450 22653 leader_election.cc:302] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:47.848977 27582 raft_consensus.cc:2781] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:47.849058 27582 raft_consensus.cc:686] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:47.849172 27582 consensus_queue.cc:237] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:47.849665 22716 raft_consensus.cc:1203] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 07:58:47.850457 24872 raft_consensus.cc:1203] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:47.851583 22444 catalog_manager.cc:5484] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:47.896883 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:47.901712 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 8e70e35cb8da46c7b8612266d21f7ac9 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=5720ff0cb45c41739920a50502ed7847]), partition=
I20240502 07:58:47.901788 24983 tablet_service.cc:1467] Processing CreateTablet for tablet 8e70e35cb8da46c7b8612266d21f7ac9 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=5720ff0cb45c41739920a50502ed7847]), partition=
I20240502 07:58:47.901988 24983 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8e70e35cb8da46c7b8612266d21f7ac9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.901978 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8e70e35cb8da46c7b8612266d21f7ac9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.904179 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 8e70e35cb8da46c7b8612266d21f7ac9 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=5720ff0cb45c41739920a50502ed7847]), partition=
I20240502 07:58:47.904397 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8e70e35cb8da46c7b8612266d21f7ac9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.903963 28047 tablet_bootstrap.cc:492] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:47.906366 28045 tablet_bootstrap.cc:492] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:47.907053 28047 tablet_bootstrap.cc:654] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.907207 28045 tablet_bootstrap.cc:654] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.907945 28047 tablet_bootstrap.cc:492] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:47.908066 28047 ts_tablet_manager.cc:1367] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.002s
I20240502 07:58:47.908455 28045 tablet_bootstrap.cc:492] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:47.908521 28045 ts_tablet_manager.cc:1367] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:47.908465 28047 raft_consensus.cc:348] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:47.908648 28047 raft_consensus.cc:374] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.908695 28047 raft_consensus.cc:729] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.908843 28047 consensus_queue.cc:260] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:47.908998 28045 raft_consensus.cc:348] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:47.909098 28045 raft_consensus.cc:374] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.909129 28045 raft_consensus.cc:729] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.909229 28045 consensus_queue.cc:260] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:47.909478 28045 ts_tablet_manager.cc:1398] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:47.909984 28047 ts_tablet_manager.cc:1398] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:47.910202 28060 tablet_bootstrap.cc:492] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:47.911864 28060 tablet_bootstrap.cc:654] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:47.914937 27599 ts_tablet_manager.cc:902] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:47.915396 27599 tablet_copy_client.cc:308] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33387
I20240502 07:58:47.915831 22737 tablet_copy_service.cc:140] P 725969d490a04fdfb3ba890c3f5096e3: Received BeginTabletCopySession request for tablet 55fb5c7475b14e0eaced5b8de1792641 from peer 4654231dfe994c1a9d2e37d9c3d75699 ({username='slave'} at 127.16.85.135:38693)
I20240502 07:58:47.915972 22737 tablet_copy_service.cc:161] P 725969d490a04fdfb3ba890c3f5096e3: Beginning new tablet copy session on tablet 55fb5c7475b14e0eaced5b8de1792641 from peer 4654231dfe994c1a9d2e37d9c3d75699 at {username='slave'} at 127.16.85.135:38693: session id = 4654231dfe994c1a9d2e37d9c3d75699-55fb5c7475b14e0eaced5b8de1792641
I20240502 07:58:47.917204 22737 tablet_copy_source_session.cc:205] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:47.918063 27599 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 55fb5c7475b14e0eaced5b8de1792641. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.919799 27599 tablet_copy_client.cc:783] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:47.919947 27599 tablet_copy_client.cc:647] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:47.921420 27599 tablet_copy_client.cc:523] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:47.922390 27599 tablet_bootstrap.cc:492] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:47.925453 28060 tablet_bootstrap.cc:492] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:47.925580 28060 ts_tablet_manager.cc:1367] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.015s user 0.002s sys 0.000s
I20240502 07:58:47.926002 28060 raft_consensus.cc:348] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:47.926149 28060 raft_consensus.cc:374] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:47.926213 28060 raft_consensus.cc:729] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:47.926380 28060 consensus_queue.cc:260] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:47.926635 28060 ts_tablet_manager.cc:1398] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:47.941982 27599 tablet_bootstrap.cc:492] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:47.942225 27599 tablet_bootstrap.cc:492] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap complete.
I20240502 07:58:47.942375 27599 ts_tablet_manager.cc:1367] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.020s user 0.018s sys 0.000s
I20240502 07:58:47.942354 27110 raft_consensus.cc:482] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:47.942454 27110 raft_consensus.cc:504] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.942838 27110 leader_election.cc:288] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:47.942766 27599 raft_consensus.cc:348] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:47.942912 27599 raft_consensus.cc:729] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: LEARNER
I20240502 07:58:47.943018 27599 consensus_queue.cc:260] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:47.943135 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "05c1913e62aa4be388888d6fa65c8523" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:47.943259 25134 raft_consensus.cc:2454] T 05c1913e62aa4be388888d6fa65c8523 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:47.943300 27599 ts_tablet_manager.cc:1398] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:47.943302 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "05c1913e62aa4be388888d6fa65c8523" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:47.943432 25266 raft_consensus.cc:2454] T 05c1913e62aa4be388888d6fa65c8523 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:47.943543 24808 leader_election.cc:302] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:47.943885 27110 raft_consensus.cc:2781] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:47.943949 27110 raft_consensus.cc:482] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:47.944008 27110 raft_consensus.cc:3037] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.944761 27110 raft_consensus.cc:504] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.945063 27110 leader_election.cc:288] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:47.945245 22737 tablet_copy_service.cc:342] P 725969d490a04fdfb3ba890c3f5096e3: Request end of tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-55fb5c7475b14e0eaced5b8de1792641 received from {username='slave'} at 127.16.85.135:38693
I20240502 07:58:47.945310 22737 tablet_copy_service.cc:433] P 725969d490a04fdfb3ba890c3f5096e3: ending tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-55fb5c7475b14e0eaced5b8de1792641 on tablet 55fb5c7475b14e0eaced5b8de1792641 with peer 4654231dfe994c1a9d2e37d9c3d75699
I20240502 07:58:47.945412 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "05c1913e62aa4be388888d6fa65c8523" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:47.945451 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "05c1913e62aa4be388888d6fa65c8523" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:47.945508 25266 raft_consensus.cc:3037] T 05c1913e62aa4be388888d6fa65c8523 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.945525 25134 raft_consensus.cc:3037] T 05c1913e62aa4be388888d6fa65c8523 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.946348 25134 raft_consensus.cc:2454] T 05c1913e62aa4be388888d6fa65c8523 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:47.946348 25266 raft_consensus.cc:2454] T 05c1913e62aa4be388888d6fa65c8523 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:47.946595 24808 leader_election.cc:302] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:47.946777 27110 raft_consensus.cc:2781] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:47.946856 27110 raft_consensus.cc:686] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:58:47.946969 27110 consensus_queue.cc:237] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:47.948453 22440 catalog_manager.cc:5484] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:47.952908 25670 raft_consensus.cc:1050] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 to VOTER
I20240502 07:58:47.953207 25670 consensus_queue.cc:237] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:47.954406 25134 raft_consensus.cc:1261] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:47.954413 22845 raft_consensus.cc:1261] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:47.954427 22586 raft_consensus.cc:1261] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:47.954761 28071 consensus_queue.cc:1035] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:47.954911 25670 consensus_queue.cc:1035] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:47.955037 27310 consensus_queue.cc:1035] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:47.956854 27333 raft_consensus.cc:2932] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:47.957193 22586 raft_consensus.cc:2932] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:47.957290 22845 raft_consensus.cc:2932] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:47.959681 22440 catalog_manager.cc:5484] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:47.960005 25003 raft_consensus.cc:1203] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 07:58:47.961031 25135 raft_consensus.cc:2932] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:47.966790 22979 consensus_queue.cc:237] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:47.967633 25135 raft_consensus.cc:1261] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:47.967631 22586 raft_consensus.cc:1261] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:47.968108 28071 consensus_queue.cc:1035] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:47.968340 25670 consensus_queue.cc:1035] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:47.969686 25670 raft_consensus.cc:2932] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:47.971606 22586 raft_consensus.cc:2932] T 28d5d20938a4472caf029c8d1ac89b82 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:47.972357 22845 raft_consensus.cc:1203] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 07:58:47.973130 22442 catalog_manager.cc:5484] T 28d5d20938a4472caf029c8d1ac89b82 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:47.974855 22429 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 28d5d20938a4472caf029c8d1ac89b82 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:47.975229 25135 raft_consensus.cc:2932] T 28d5d20938a4472caf029c8d1ac89b82 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:47.975878 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet 28d5d20938a4472caf029c8d1ac89b82 with delete_type TABLET_DATA_TOMBSTONED (TS 7db4178eed174225bfb24f5f9445ce88 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:47.976949 27671 ts_tablet_manager.cc:902] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88: Initiating tablet copy from peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:47.977092 28071 raft_consensus.cc:482] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:47.977165 28071 raft_consensus.cc:504] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.977406 27671 tablet_copy_client.cc:308] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.133:35459
I20240502 07:58:47.977797 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f55942f203d455394d2f81f9ae91f0f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:47.977900 25135 raft_consensus.cc:2454] T 9f55942f203d455394d2f81f9ae91f0f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:47.977912 24892 tablet_copy_service.cc:140] P 6d02017318a2461796d431962fe6a537: Received BeginTabletCopySession request for tablet 3bb108b6ab55461fb9fabf431d953ef2 from peer 7db4178eed174225bfb24f5f9445ce88 ({username='slave'} at 127.16.85.131:54881)
I20240502 07:58:47.978016 24892 tablet_copy_service.cc:161] P 6d02017318a2461796d431962fe6a537: Beginning new tablet copy session on tablet 3bb108b6ab55461fb9fabf431d953ef2 from peer 7db4178eed174225bfb24f5f9445ce88 at {username='slave'} at 127.16.85.131:54881: session id = 7db4178eed174225bfb24f5f9445ce88-3bb108b6ab55461fb9fabf431d953ef2
I20240502 07:58:47.978139 28084 tablet_replica.cc:330] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:47.978139 22915 leader_election.cc:302] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 4654231dfe994c1a9d2e37d9c3d75699; no voters:
I20240502 07:58:47.978281 25670 raft_consensus.cc:2781] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:47.978329 25670 raft_consensus.cc:482] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:47.978360 25670 raft_consensus.cc:3037] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.978458 28084 raft_consensus.cc:2229] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:47.978657 28084 raft_consensus.cc:2258] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:47.978960 24892 tablet_copy_source_session.cc:205] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:47.979382 28084 ts_tablet_manager.cc:1875] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:47.980237 27671 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3bb108b6ab55461fb9fabf431d953ef2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:47.980274 25670 raft_consensus.cc:504] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.980588 25670 leader_election.cc:288] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:47.980947 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f55942f203d455394d2f81f9ae91f0f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:47.981050 24872 raft_consensus.cc:3037] T 9f55942f203d455394d2f81f9ae91f0f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.981150 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f55942f203d455394d2f81f9ae91f0f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:47.981232 25135 raft_consensus.cc:3037] T 9f55942f203d455394d2f81f9ae91f0f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:47.981963 24872 raft_consensus.cc:2454] T 9f55942f203d455394d2f81f9ae91f0f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:47.982057 25135 raft_consensus.cc:2454] T 9f55942f203d455394d2f81f9ae91f0f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:47.982250 27671 tablet_copy_client.cc:783] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:47.982286 22915 leader_election.cc:302] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 4654231dfe994c1a9d2e37d9c3d75699; no voters:
I20240502 07:58:47.982383 27671 tablet_copy_client.cc:647] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:47.982482 25670 raft_consensus.cc:2781] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:47.982542 28084 ts_tablet_manager.cc:1888] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:47.982609 28084 log.cc:1196] T 28d5d20938a4472caf029c8d1ac89b82 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/28d5d20938a4472caf029c8d1ac89b82
I20240502 07:58:47.983068 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 28d5d20938a4472caf029c8d1ac89b82 (table rebalance_test_table_11 [id=0c1156cac15d48028ebf5c07c67ea832]) successfully deleted
I20240502 07:58:47.983523 25670 raft_consensus.cc:686] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:47.983654 25670 consensus_queue.cc:237] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:47.984248 27671 tablet_copy_client.cc:523] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:47.984254 28071 leader_election.cc:288] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:47.984623 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f55942f203d455394d2f81f9ae91f0f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:47.984799 24872 raft_consensus.cc:2362] T 9f55942f203d455394d2f81f9ae91f0f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1. Re-sending same reply.
I20240502 07:58:47.985610 22435 catalog_manager.cc:5484] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:47.985816 27671 tablet_bootstrap.cc:492] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:47.989675 27671 tablet_bootstrap.cc:492] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:47.989907 27671 tablet_bootstrap.cc:492] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap complete.
I20240502 07:58:47.990056 27671 ts_tablet_manager.cc:1367] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 07:58:47.990387 27671 raft_consensus.cc:348] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 07:58:47.990473 27671 raft_consensus.cc:729] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: LEARNER
I20240502 07:58:47.990563 27671 consensus_queue.cc:260] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 07:58:47.990963 27671 ts_tablet_manager.cc:1398] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 07:58:47.991397 24892 tablet_copy_service.cc:342] P 6d02017318a2461796d431962fe6a537: Request end of tablet copy session 7db4178eed174225bfb24f5f9445ce88-3bb108b6ab55461fb9fabf431d953ef2 received from {username='slave'} at 127.16.85.131:54881
I20240502 07:58:47.991463 24892 tablet_copy_service.cc:433] P 6d02017318a2461796d431962fe6a537: ending tablet copy session 7db4178eed174225bfb24f5f9445ce88-3bb108b6ab55461fb9fabf431d953ef2 on tablet 3bb108b6ab55461fb9fabf431d953ef2 with peer 7db4178eed174225bfb24f5f9445ce88
I20240502 07:58:48.012913 26079 raft_consensus.cc:1050] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88: attempting to promote NON_VOTER 6d02017318a2461796d431962fe6a537 to VOTER
I20240502 07:58:48.013265 26079 consensus_queue.cc:237] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:48.014665 22979 raft_consensus.cc:1261] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.014757 24872 raft_consensus.cc:1261] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.014818 22716 raft_consensus.cc:1261] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.015220 28118 consensus_queue.cc:1035] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.015452 25532 consensus_queue.cc:1035] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.015647 26284 consensus_queue.cc:1035] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.019591 25532 raft_consensus.cc:2932] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.021522 22979 raft_consensus.cc:2932] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.022261 22716 raft_consensus.cc:2932] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.022547 22435 catalog_manager.cc:5484] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:48.022822 24872 raft_consensus.cc:2932] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.025410 22845 consensus_queue.cc:237] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:48.026818 22716 raft_consensus.cc:1261] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:48.027976 28118 consensus_queue.cc:1035] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:48.030015 28118 raft_consensus.cc:2932] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.031677 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4363e32404304568b94f615e4e29f883 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:48.032557 22716 raft_consensus.cc:2932] T 4363e32404304568b94f615e4e29f883 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.033289 24872 raft_consensus.cc:1261] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:48.033851 25532 consensus_queue.cc:1035] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:48.034189 22435 catalog_manager.cc:5484] T 4363e32404304568b94f615e4e29f883 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:48.034806 24872 raft_consensus.cc:2932] T 4363e32404304568b94f615e4e29f883 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.037777 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet 4363e32404304568b94f615e4e29f883 with delete_type TABLET_DATA_TOMBSTONED (TS 02a92e1c084b4308b9e855a4cc2c295f not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:48.041138 28131 tablet_replica.cc:330] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:48.041342 28131 raft_consensus.cc:2229] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:48.041671 28131 raft_consensus.cc:2258] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:48.044332 25266 raft_consensus.cc:1203] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 07:58:48.047873 28131 ts_tablet_manager.cc:1875] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:48.050022 28131 ts_tablet_manager.cc:1888] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:48.050114 28131 log.cc:1196] T 4363e32404304568b94f615e4e29f883 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/4363e32404304568b94f615e4e29f883
I20240502 07:58:48.050627 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 4363e32404304568b94f615e4e29f883 (table rebalance_test_table_17 [id=35a66cc6bd364d818b35bceada233a98]) successfully deleted
I20240502 07:58:48.065063 28154 ts_tablet_manager.cc:902] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537: Initiating tablet copy from peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:48.065644 28154 tablet_copy_client.cc:308] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:37451
I20240502 07:58:48.066061 22606 tablet_copy_service.cc:140] P 8bea0fe8d8044a2685f92f7a10c9cf85: Received BeginTabletCopySession request for tablet c53e8b3bd774466dbafe4994472975bc from peer 6d02017318a2461796d431962fe6a537 ({username='slave'} at 127.16.85.133:57895)
I20240502 07:58:48.066159 22606 tablet_copy_service.cc:161] P 8bea0fe8d8044a2685f92f7a10c9cf85: Beginning new tablet copy session on tablet c53e8b3bd774466dbafe4994472975bc from peer 6d02017318a2461796d431962fe6a537 at {username='slave'} at 127.16.85.133:57895: session id = 6d02017318a2461796d431962fe6a537-c53e8b3bd774466dbafe4994472975bc
I20240502 07:58:48.066898 22606 tablet_copy_source_session.cc:205] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:48.067713 28154 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c53e8b3bd774466dbafe4994472975bc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.069769 28154 tablet_copy_client.cc:783] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:48.069972 28154 tablet_copy_client.cc:647] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:48.071063 28154 tablet_copy_client.cc:523] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:48.072137 28154 tablet_bootstrap.cc:492] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:48.095495 22586 consensus_queue.cc:237] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:48.095968 28154 tablet_bootstrap.cc:492] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:48.096210 28154 tablet_bootstrap.cc:492] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537: Bootstrap complete.
I20240502 07:58:48.096369 28154 ts_tablet_manager.cc:1367] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.024s user 0.017s sys 0.004s
I20240502 07:58:48.096746 28154 raft_consensus.cc:348] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:48.096853 28154 raft_consensus.cc:729] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: LEARNER
I20240502 07:58:48.096870 25135 raft_consensus.cc:1261] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 07:58:48.096912 22845 raft_consensus.cc:1261] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 07:58:48.096951 28154 consensus_queue.cc:260] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:48.097200 28154 ts_tablet_manager.cc:1398] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.004s sys 0.000s
I20240502 07:58:48.097216 27136 consensus_queue.cc:1035] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:48.097458 27136 consensus_queue.cc:1035] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:48.097697 22606 tablet_copy_service.cc:342] P 8bea0fe8d8044a2685f92f7a10c9cf85: Request end of tablet copy session 6d02017318a2461796d431962fe6a537-c53e8b3bd774466dbafe4994472975bc received from {username='slave'} at 127.16.85.133:57895
I20240502 07:58:48.097769 22606 tablet_copy_service.cc:433] P 8bea0fe8d8044a2685f92f7a10c9cf85: ending tablet copy session 6d02017318a2461796d431962fe6a537-c53e8b3bd774466dbafe4994472975bc on tablet c53e8b3bd774466dbafe4994472975bc with peer 6d02017318a2461796d431962fe6a537
W20240502 07:58:48.097940 22521 consensus_peers.cc:485] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Couldn't send request to peer 7eced220cf1642f3873fc385b6191e69. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: dd815609d7324cc588f02b6cb40cfbfd. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:48.099236 27136 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:48.100069 25135 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:48.101090 22845 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:48.102523 22442 catalog_manager.cc:5484] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:48.105480 26697 raft_consensus.cc:482] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:48.105572 26697 raft_consensus.cc:504] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:48.106020 26697 leader_election.cc:288] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:48.106441 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8e70e35cb8da46c7b8612266d21f7ac9" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:48.106451 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8e70e35cb8da46c7b8612266d21f7ac9" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:48.106540 22586 raft_consensus.cc:2454] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:48.106549 22979 raft_consensus.cc:2454] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:48.106817 24937 leader_election.cc:302] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:48.107121 26697 raft_consensus.cc:2781] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:48.107173 26697 raft_consensus.cc:482] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:48.107215 26697 raft_consensus.cc:3037] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:48.107889 26697 raft_consensus.cc:504] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:48.108225 26697 leader_election.cc:288] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:48.108438 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8e70e35cb8da46c7b8612266d21f7ac9" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:48.108542 22979 raft_consensus.cc:3037] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:48.108597 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8e70e35cb8da46c7b8612266d21f7ac9" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:48.108707 22586 raft_consensus.cc:3037] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:48.109309 22979 raft_consensus.cc:2454] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:48.109514 22586 raft_consensus.cc:2454] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:48.109525 24937 leader_election.cc:302] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:48.109674 26697 raft_consensus.cc:2781] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:48.109740 26697 raft_consensus.cc:686] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Becoming Leader. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Running, Role: LEADER
I20240502 07:58:48.109841 26697 consensus_queue.cc:237] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:48.111398 22435 catalog_manager.cc:5484] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: term changed from 0 to 1, leader changed from <none> to 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134). New cstate: current_term: 1 leader_uuid: "74f9aa91c2c448f980b512fe2e8d672e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:48.114255 25135 consensus_queue.cc:237] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 07:58:48.115520 25003 raft_consensus.cc:1261] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:48.115561 22586 raft_consensus.cc:1261] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
W20240502 07:58:48.115880 25069 consensus_peers.cc:485] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): Couldn't send request to peer 02a92e1c084b4308b9e855a4cc2c295f. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 5487e66cb4d8416094a1321cb989f018. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:48.115967 27488 consensus_queue.cc:1035] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:48.116106 26764 consensus_queue.cc:1035] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:48.117591 27488 raft_consensus.cc:2932] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 07:58:48.118057 25003 raft_consensus.cc:2932] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 07:58:48.118199 22586 raft_consensus.cc:2932] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 07:58:48.120327 22440 catalog_manager.cc:5484] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: config changed from index -1 to 2, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 07:58:48.123540 27808 raft_consensus.cc:1050] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69: attempting to promote NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f to VOTER
I20240502 07:58:48.123944 27808 consensus_queue.cc:237] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } }
I20240502 07:58:48.125293 24872 raft_consensus.cc:1261] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.125289 22979 raft_consensus.cc:1261] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.125435 25003 raft_consensus.cc:1261] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.125661 27495 consensus_queue.cc:1035] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.125810 27808 consensus_queue.cc:1035] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.125905 27002 consensus_queue.cc:1035] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.128000 28205 raft_consensus.cc:2932] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } } }
I20240502 07:58:48.128465 22979 raft_consensus.cc:2932] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } } }
I20240502 07:58:48.129850 25003 raft_consensus.cc:2932] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } } }
I20240502 07:58:48.131980 24872 raft_consensus.cc:2932] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } } }
I20240502 07:58:48.132033 22442 catalog_manager.cc:5484] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 2 to 3, 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } } }
I20240502 07:58:48.136556 25266 consensus_queue.cc:237] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } }
I20240502 07:58:48.137775 22979 raft_consensus.cc:1261] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:48.137840 24872 raft_consensus.cc:1261] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:48.138200 27002 consensus_queue.cc:1035] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:48.138389 27808 consensus_queue.cc:1035] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:48.139783 27002 raft_consensus.cc:2932] T 4e687b2e36574856a45752104ce83953 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } } }
I20240502 07:58:48.140157 22979 raft_consensus.cc:2932] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } } }
I20240502 07:58:48.141239 24872 raft_consensus.cc:2932] T 4e687b2e36574856a45752104ce83953 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } } }
I20240502 07:58:48.141964 22430 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4e687b2e36574856a45752104ce83953 with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:48.142767 22436 catalog_manager.cc:5484] T 4e687b2e36574856a45752104ce83953 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 3 to 4, VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } } }
I20240502 07:58:48.144941 24983 tablet_service.cc:1514] Processing DeleteTablet for tablet 4e687b2e36574856a45752104ce83953 with delete_type TABLET_DATA_TOMBSTONED (TS 74f9aa91c2c448f980b512fe2e8d672e not found in new config with opid_index 4) from {username='slave'} at 127.0.0.1:42878
I20240502 07:58:48.145157 28043 tablet_replica.cc:330] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e: stopping tablet replica
I20240502 07:58:48.145280 28043 raft_consensus.cc:2229] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:48.145388 28043 raft_consensus.cc:2258] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:48.145889 28043 ts_tablet_manager.cc:1875] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:48.147598 28043 ts_tablet_manager.cc:1888] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.3
I20240502 07:58:48.147640 28043 log.cc:1196] T 4e687b2e36574856a45752104ce83953 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/wals/4e687b2e36574856a45752104ce83953
I20240502 07:58:48.148067 22430 catalog_manager.cc:4830] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): tablet 4e687b2e36574856a45752104ce83953 (table rebalancer_extra_table_6 [id=4aefdd54f1e145688b445ce3d02fb21e]) successfully deleted
I20240502 07:58:48.222362 22979 consensus_queue.cc:237] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:48.223682 25266 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 07:58:48.223713 22847 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 07:58:48.223891 22912 consensus_peers.cc:485] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): Couldn't send request to peer 6d02017318a2461796d431962fe6a537. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ca85b40df34d4e1c91a55d7dd69e1494. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:48.224081 27333 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:48.224251 25670 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:48.225757 27333 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:48.226161 22847 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:48.226241 25266 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:48.228526 22435 catalog_manager.cc:5484] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 6 to 7, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:48.237188 22716 consensus_queue.cc:237] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
W20240502 07:58:48.238626 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: da39029564d042f6acfc23f99fb63dfd. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:48.238655 22979 raft_consensus.cc:1261] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:48.238708 25135 raft_consensus.cc:1261] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:48.239005 26995 consensus_queue.cc:1035] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:48.239157 27582 consensus_queue.cc:1035] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:48.240795 27303 raft_consensus.cc:2932] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:48.241314 22979 raft_consensus.cc:2932] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:48.242475 25135 raft_consensus.cc:2932] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:48.243958 22436 catalog_manager.cc:5484] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index -1 to 2, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:48.251106 22435 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:48.254966 22979 raft_consensus.cc:1261] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:48.255309 22586 raft_consensus.cc:1261] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:48.255419 26696 consensus_queue.cc:1035] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:48.255764 26696 consensus_queue.cc:1035] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:48.258138 28320 tablet.cc:1722] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:48.258865 22435 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet 8e70e35cb8da46c7b8612266d21f7ac9 (table rebalancer_dynamic_table [id=5720ff0cb45c41739920a50502ed7847]). Expected version 1 got 0
I20240502 07:58:48.259037 28320 tablet.cc:1620] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e: MemRowSet was empty: no flush needed.
I20240502 07:58:48.259897 22440 catalog_manager.cc:5576] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451) does not have the latest schema for tablet 8e70e35cb8da46c7b8612266d21f7ac9 (table rebalancer_dynamic_table [id=5720ff0cb45c41739920a50502ed7847]). Expected version 1 got 0
I20240502 07:58:48.260087 28321 tablet.cc:1722] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:48.260579 28322 tablet.cc:1722] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:48.260833 28321 tablet.cc:1620] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:58:48.261307 28322 tablet.cc:1620] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:58:48.262889 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=5720ff0cb45c41739920a50502ed7847] alter complete (version 1)
I20240502 07:58:48.263420 22440 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet 8e70e35cb8da46c7b8612266d21f7ac9 (table rebalancer_dynamic_table [id=5720ff0cb45c41739920a50502ed7847]). Expected version 1 got 0
I20240502 07:58:48.274868 27110 raft_consensus.cc:1050] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537: attempting to promote NON_VOTER 7db4178eed174225bfb24f5f9445ce88 to VOTER
I20240502 07:58:48.275337 27110 consensus_queue.cc:237] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } }
I20240502 07:58:48.276970 25003 raft_consensus.cc:1261] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.277004 22847 raft_consensus.cc:1261] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.277101 22586 raft_consensus.cc:1261] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.277326 27338 consensus_queue.cc:1035] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.277460 27339 consensus_queue.cc:1035] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.277747 27339 consensus_queue.cc:1035] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.281088 27338 raft_consensus.cc:2932] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:48.282110 25003 raft_consensus.cc:2932] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:48.283632 22586 raft_consensus.cc:2932] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:48.283676 22847 raft_consensus.cc:2932] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:48.284009 22436 catalog_manager.cc:5484] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 reported cstate change: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:48.287492 25670 raft_consensus.cc:1050] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 to VOTER
I20240502 07:58:48.287457 24872 consensus_queue.cc:237] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } }
I20240502 07:58:48.287870 25670 consensus_queue.cc:237] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:48.288889 22847 raft_consensus.cc:1261] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:48.288923 25003 raft_consensus.cc:1261] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:48.289242 27110 consensus_queue.cc:1035] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:48.289286 22716 raft_consensus.cc:1261] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:48.289381 25266 raft_consensus.cc:1261] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:48.289615 27310 consensus_queue.cc:1035] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:48.289777 27333 consensus_queue.cc:1035] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:48.290177 27110 consensus_queue.cc:1035] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:48.290416 25135 raft_consensus.cc:1261] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:48.290781 27310 consensus_queue.cc:1035] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:48.291090 27110 raft_consensus.cc:2932] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:48.292730 27310 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.292858 22428 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a071d3a1489e4d97ae6ceea4b0478459 with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:48.293135 25003 raft_consensus.cc:2932] T a071d3a1489e4d97ae6ceea4b0478459 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:48.293260 22847 raft_consensus.cc:2932] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:48.293450 22440 catalog_manager.cc:5484] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 reported cstate change: config changed from index 3 to 4, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:48.295634 22436 catalog_manager.cc:5484] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 7 to 8, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:48.296358 25266 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.296787 25135 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.297235 22716 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.299122 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet a071d3a1489e4d97ae6ceea4b0478459 with delete_type TABLET_DATA_TOMBSTONED (TS 8bea0fe8d8044a2685f92f7a10c9cf85 not found in new config with opid_index 4) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:48.299444 28352 tablet_replica.cc:330] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:48.299590 28352 raft_consensus.cc:2229] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:48.299731 28352 raft_consensus.cc:2258] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:48.300326 28352 ts_tablet_manager.cc:1875] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:48.302575 28352 ts_tablet_manager.cc:1888] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.3
I20240502 07:58:48.302690 28352 log.cc:1196] T a071d3a1489e4d97ae6ceea4b0478459 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/a071d3a1489e4d97ae6ceea4b0478459
I20240502 07:58:48.302682 22979 consensus_queue.cc:237] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:48.303694 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet a071d3a1489e4d97ae6ceea4b0478459 (table rebalancer_extra_table_2 [id=0c4cae853d5c4dd0a2340f80a1b1b703]) successfully deleted
I20240502 07:58:48.303759 25266 raft_consensus.cc:1261] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 07:58:48.303789 25134 raft_consensus.cc:1261] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 07:58:48.304073 25670 consensus_queue.cc:1035] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:48.304235 28071 consensus_queue.cc:1035] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:48.305768 25670 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.306126 25135 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.307365 25266 raft_consensus.cc:2932] T cfff02438656430686be0f03f5c0b8ab P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.307790 22429 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cfff02438656430686be0f03f5c0b8ab with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:48.308029 22440 catalog_manager.cc:5484] T cfff02438656430686be0f03f5c0b8ab P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 8 to 9, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:48.310493 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet cfff02438656430686be0f03f5c0b8ab with delete_type TABLET_DATA_TOMBSTONED (TS 725969d490a04fdfb3ba890c3f5096e3 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:48.310909 28353 tablet_replica.cc:330] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:48.311070 28353 raft_consensus.cc:2229] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:48.311581 28353 raft_consensus.cc:2258] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:48.312245 28353 ts_tablet_manager.cc:1875] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:48.314913 27303 consensus_queue.cc:1035] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:48.315217 28353 ts_tablet_manager.cc:1888] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 07:58:48.315285 28353 log.cc:1196] T cfff02438656430686be0f03f5c0b8ab P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/cfff02438656430686be0f03f5c0b8ab
I20240502 07:58:48.316131 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet cfff02438656430686be0f03f5c0b8ab (table rebalance_test_table_9 [id=bc838e66a1294569a905266b5619a7d7]) successfully deleted
I20240502 07:58:48.318994 27303 consensus_queue.cc:1035] T 0506cd627d684ac784a5c738f87389af P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:48.325871 25003 raft_consensus.cc:1203] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:48.335587 26079 raft_consensus.cc:1050] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88: attempting to promote NON_VOTER 6d02017318a2461796d431962fe6a537 to VOTER
I20240502 07:58:48.335980 26079 consensus_queue.cc:237] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:48.337262 22586 raft_consensus.cc:1261] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.337604 28118 consensus_queue.cc:1035] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.337699 22715 raft_consensus.cc:1261] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.337689 24872 raft_consensus.cc:1261] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.338152 28118 consensus_queue.cc:1035] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.339210 26079 consensus_queue.cc:1035] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.340499 25532 raft_consensus.cc:2932] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.340963 22715 raft_consensus.cc:2932] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.342638 22586 raft_consensus.cc:2932] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.343008 24872 raft_consensus.cc:2932] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.343195 22440 catalog_manager.cc:5484] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:48.347872 22847 consensus_queue.cc:237] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:48.348940 24872 raft_consensus.cc:1261] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:48.349248 22586 raft_consensus.cc:1261] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:48.349304 28118 consensus_queue.cc:1035] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:48.349593 28118 consensus_queue.cc:1035] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:48.351023 26079 raft_consensus.cc:2932] T ee06ff655f9c46c1bb06af2dd539fde7 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.351406 24872 raft_consensus.cc:2932] T ee06ff655f9c46c1bb06af2dd539fde7 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.351771 22586 raft_consensus.cc:2932] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.353487 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ee06ff655f9c46c1bb06af2dd539fde7 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:48.354498 22444 catalog_manager.cc:5484] T ee06ff655f9c46c1bb06af2dd539fde7 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.356565 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet ee06ff655f9c46c1bb06af2dd539fde7 with delete_type TABLET_DATA_TOMBSTONED (TS 725969d490a04fdfb3ba890c3f5096e3 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:48.356794 28353 tablet_replica.cc:330] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:48.356938 28353 raft_consensus.cc:2229] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:48.357081 28353 raft_consensus.cc:2258] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:48.358402 28353 ts_tablet_manager.cc:1875] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:48.360265 28353 ts_tablet_manager.cc:1888] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:48.360322 28353 log.cc:1196] T ee06ff655f9c46c1bb06af2dd539fde7 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/ee06ff655f9c46c1bb06af2dd539fde7
I20240502 07:58:48.360798 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet ee06ff655f9c46c1bb06af2dd539fde7 (table rebalance_test_table_9 [id=bc838e66a1294569a905266b5619a7d7]) successfully deleted
I20240502 07:58:48.367105 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:48.371932 28320 tablet.cc:1722] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:48.372254 28320 tablet.cc:1620] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e: MemRowSet was empty: no flush needed.
I20240502 07:58:48.372335 28322 tablet.cc:1722] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:48.372620 28322 tablet.cc:1620] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:58:48.374550 28321 tablet.cc:1722] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:48.374816 28321 tablet.cc:1620] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:58:48.377986 22435 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=5720ff0cb45c41739920a50502ed7847] alter complete (version 2)
I20240502 07:58:48.386117 28118 raft_consensus.cc:1050] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88: attempting to promote NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 to VOTER
I20240502 07:58:48.386533 28118 consensus_queue.cc:237] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:48.388103 25135 raft_consensus.cc:1261] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.388460 22584 raft_consensus.cc:1261] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.388912 25532 consensus_queue.cc:1035] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.389968 25532 consensus_queue.cc:1035] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.391733 22978 raft_consensus.cc:1261] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.392112 25532 consensus_queue.cc:1035] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.393069 25532 raft_consensus.cc:2932] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.394047 22978 raft_consensus.cc:2932] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.395336 22584 raft_consensus.cc:2932] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.396034 22444 catalog_manager.cc:5484] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:48.398624 25134 raft_consensus.cc:2932] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.403282 22847 consensus_queue.cc:237] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:48.404615 25134 raft_consensus.cc:1261] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:48.404949 22584 raft_consensus.cc:1261] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:48.405328 28118 consensus_queue.cc:1035] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:48.405464 26079 consensus_queue.cc:1035] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:48.407753 25532 raft_consensus.cc:2932] T fcbab4ec41a74ff8b0c564ac7a0b044e P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.408951 25134 raft_consensus.cc:2932] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.409669 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet fcbab4ec41a74ff8b0c564ac7a0b044e with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:48.409950 22584 raft_consensus.cc:2932] T fcbab4ec41a74ff8b0c564ac7a0b044e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.411440 22444 catalog_manager.cc:5484] T fcbab4ec41a74ff8b0c564ac7a0b044e P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.413612 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet fcbab4ec41a74ff8b0c564ac7a0b044e with delete_type TABLET_DATA_TOMBSTONED (TS 02a92e1c084b4308b9e855a4cc2c295f not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:48.413832 28131 tablet_replica.cc:330] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:48.413986 28131 raft_consensus.cc:2229] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:48.414117 28131 raft_consensus.cc:2258] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:48.414759 28131 ts_tablet_manager.cc:1875] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:48.416870 28131 ts_tablet_manager.cc:1888] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:48.416944 28131 log.cc:1196] T fcbab4ec41a74ff8b0c564ac7a0b044e P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/fcbab4ec41a74ff8b0c564ac7a0b044e
I20240502 07:58:48.417140 28188 raft_consensus.cc:1050] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85: attempting to promote NON_VOTER 7db4178eed174225bfb24f5f9445ce88 to VOTER
I20240502 07:58:48.417464 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet fcbab4ec41a74ff8b0c564ac7a0b044e (table rebalance_test_table_19 [id=65deae5d69b44af784a28c39cd113241]) successfully deleted
I20240502 07:58:48.417526 28188 consensus_queue.cc:237] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } }
I20240502 07:58:48.418875 22978 raft_consensus.cc:1261] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.419173 24872 raft_consensus.cc:1261] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.419374 27136 consensus_queue.cc:1035] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.419770 27136 consensus_queue.cc:1035] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.420234 22847 raft_consensus.cc:1261] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.420563 27136 consensus_queue.cc:1035] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.421922 25532 raft_consensus.cc:1050] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88: attempting to promote NON_VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 to VOTER
I20240502 07:58:48.422284 25532 consensus_queue.cc:237] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: false } }
I20240502 07:58:48.422729 27136 raft_consensus.cc:2932] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:48.423767 25266 raft_consensus.cc:1261] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.423941 22845 raft_consensus.cc:2932] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:48.424312 28118 consensus_queue.cc:1035] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.424914 22584 raft_consensus.cc:1261] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEARNER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.425334 25134 raft_consensus.cc:1261] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.425619 22978 raft_consensus.cc:2932] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:48.425793 26284 consensus_queue.cc:1035] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.425828 22435 catalog_manager.cc:5484] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:48.428220 24872 raft_consensus.cc:2932] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:48.428824 22586 consensus_queue.cc:237] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } }
I20240502 07:58:48.430065 28118 consensus_queue.cc:1035] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.430294 22845 raft_consensus.cc:1261] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:48.430636 28188 consensus_queue.cc:1035] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:48.432442 28188 raft_consensus.cc:2932] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:48.432808 22845 raft_consensus.cc:2932] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:48.434031 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 415879c2eee74932b9ac45282f6dd0fb with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:48.435688 22978 raft_consensus.cc:1261] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:48.435760 22435 catalog_manager.cc:5484] T 415879c2eee74932b9ac45282f6dd0fb P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 3 to 4, VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) evicted. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:48.436116 24714 consensus_queue.cc:1035] T 415879c2eee74932b9ac45282f6dd0fb P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:48.437040 22978 raft_consensus.cc:2932] T 415879c2eee74932b9ac45282f6dd0fb P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:48.437817 28118 raft_consensus.cc:2932] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: false } } }
I20240502 07:58:48.439640 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet 415879c2eee74932b9ac45282f6dd0fb with delete_type TABLET_DATA_TOMBSTONED (TS 6d02017318a2461796d431962fe6a537 not found in new config with opid_index 4) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:48.439697 25266 raft_consensus.cc:2932] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: false } } }
I20240502 07:58:48.440578 28439 tablet_replica.cc:330] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:48.440773 28439 raft_consensus.cc:2229] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:48.440959 28439 raft_consensus.cc:2258] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:48.441190 25134 raft_consensus.cc:2932] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: false } } }
I20240502 07:58:48.442277 22584 raft_consensus.cc:2932] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: false } } }
I20240502 07:58:48.443138 22444 catalog_manager.cc:5484] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 2 to 3, 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:48.447408 28439 ts_tablet_manager.cc:1875] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:48.449376 28439 ts_tablet_manager.cc:1888] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.3
I20240502 07:58:48.449450 28439 log.cc:1196] T 415879c2eee74932b9ac45282f6dd0fb P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/415879c2eee74932b9ac45282f6dd0fb
I20240502 07:58:48.450037 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet 415879c2eee74932b9ac45282f6dd0fb (table rebalancer_extra_table_4 [id=61a72ea9962740bd8839987228b47853]) successfully deleted
I20240502 07:58:48.453706 27002 raft_consensus.cc:1050] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69: attempting to promote NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 to VOTER
I20240502 07:58:48.454066 27002 consensus_queue.cc:237] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false } }
I20240502 07:58:48.454538 25003 consensus_queue.cc:237] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 07:58:48.455224 22715 raft_consensus.cc:1261] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.455663 25134 raft_consensus.cc:1261] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.455966 25135 raft_consensus.cc:1261] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:48.456161 24872 raft_consensus.cc:1261] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 74f9aa91c2c448f980b512fe2e8d672e: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:48.456494 27808 consensus_queue.cc:1035] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.456791 27495 consensus_queue.cc:1035] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.457094 28444 consensus_queue.cc:1035] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:48.457281 26696 consensus_queue.cc:1035] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:48.458915 24872 raft_consensus.cc:1261] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.459390 27495 consensus_queue.cc:1035] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.459633 27303 raft_consensus.cc:1050] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3: attempting to promote NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e to VOTER
I20240502 07:58:48.459964 27303 consensus_queue.cc:237] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
W20240502 07:58:48.461539 24936 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e -> Peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): Couldn't send request to peer 725969d490a04fdfb3ba890c3f5096e3. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 90acd69ebaba4fb2a67e16172e8d7aa5. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:48.461637 22978 raft_consensus.cc:1261] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.461954 22845 raft_consensus.cc:1261] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.461915 28444 raft_consensus.cc:2932] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } } }
I20240502 07:58:48.463191 26995 consensus_queue.cc:1035] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.463888 25134 raft_consensus.cc:2932] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } } }
I20240502 07:58:48.465775 22845 raft_consensus.cc:1203] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 07:58:48.467007 22440 catalog_manager.cc:5484] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index -1 to 2, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New cstate: current_term: 1 leader_uuid: "74f9aa91c2c448f980b512fe2e8d672e" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } } }
I20240502 07:58:48.467161 24872 raft_consensus.cc:2932] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } } }
I20240502 07:58:48.468281 27495 raft_consensus.cc:2932] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false } } }
I20240502 07:58:48.468817 24872 raft_consensus.cc:2932] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false } } }
I20240502 07:58:48.469004 26995 consensus_queue.cc:1035] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.470070 25003 raft_consensus.cc:1261] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.470736 25134 raft_consensus.cc:2932] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false } } }
I20240502 07:58:48.470990 22716 raft_consensus.cc:2932] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false } } }
I20240502 07:58:48.472317 27582 consensus_queue.cc:1035] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.472398 22442 catalog_manager.cc:5484] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: config changed from index 2 to 3, 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:48.475945 25266 consensus_queue.cc:237] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false } }
I20240502 07:58:48.477124 24872 raft_consensus.cc:1261] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:48.478080 27808 consensus_queue.cc:1035] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:48.479460 26995 raft_consensus.cc:2932] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.479795 22717 consensus_queue.cc:237] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:48.479899 27808 raft_consensus.cc:2932] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false } } }
I20240502 07:58:48.481474 22845 raft_consensus.cc:1261] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:48.481549 22584 raft_consensus.cc:1261] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:48.481676 22430 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cfcaa9ee1fbe48ae91796b184ecaf2a3 with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:48.481858 27303 consensus_queue.cc:1035] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:48.481986 27582 consensus_queue.cc:1035] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:48.482153 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:48.482990 27339 consensus_queue.cc:1035] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:48.483587 25003 raft_consensus.cc:2932] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.484225 24871 raft_consensus.cc:2932] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false } } }
W20240502 07:58:48.485577 22650 consensus_peers.cc:485] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): Couldn't send request to peer 6d02017318a2461796d431962fe6a537. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 023958ce03b84b2da063c13e4284e04d. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:48.486519 27339 consensus_queue.cc:1035] T 05c1913e62aa4be388888d6fa65c8523 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:48.486560 22444 catalog_manager.cc:5484] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 6d02017318a2461796d431962fe6a537 reported cstate change: config changed from index 3 to 4, VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) evicted. New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false } } }
I20240502 07:58:48.487248 22442 catalog_manager.cc:5484] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.489953 28320 tablet.cc:1722] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:48.490589 25115 tablet_service.cc:1514] Processing DeleteTablet for tablet cfcaa9ee1fbe48ae91796b184ecaf2a3 with delete_type TABLET_DATA_TOMBSTONED (TS 4654231dfe994c1a9d2e37d9c3d75699 not found in new config with opid_index 4) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:48.490955 28469 tablet_replica.cc:330] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:48.491150 28469 raft_consensus.cc:2229] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:48.491343 28469 raft_consensus.cc:2258] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:48.491616 28322 tablet.cc:1722] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:48.493324 28321 tablet.cc:1722] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:48.493638 22435 catalog_manager.cc:5031] ChangeConfig:REMOVE_PEER RPC for tablet cfcaa9ee1fbe48ae91796b184ecaf2a3 with cas_config_opid_index 3: aborting the task: latest config opid_index 4; task opid_index 3
I20240502 07:58:48.494858 22979 raft_consensus.cc:2932] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.496877 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=5720ff0cb45c41739920a50502ed7847] alter complete (version 3)
I20240502 07:58:48.497668 22716 raft_consensus.cc:1261] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:48.498139 27495 consensus_queue.cc:1035] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:48.499101 28469 ts_tablet_manager.cc:1875] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:48.499015 22716 raft_consensus.cc:2932] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false } } }
I20240502 07:58:48.501364 28469 ts_tablet_manager.cc:1888] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.3
I20240502 07:58:48.501659 22435 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_16"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:48.502050 28469 log.cc:1196] T cfcaa9ee1fbe48ae91796b184ecaf2a3 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/cfcaa9ee1fbe48ae91796b184ecaf2a3
I20240502 07:58:48.502557 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet cfcaa9ee1fbe48ae91796b184ecaf2a3 (table rebalancer_extra_table_2 [id=0c4cae853d5c4dd0a2340f80a1b1b703]) successfully deleted
I20240502 07:58:48.502818 27582 raft_consensus.cc:2932] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:48.503111 22847 raft_consensus.cc:2932] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.504868 22584 raft_consensus.cc:2932] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:48.504866 22845 raft_consensus.cc:2932] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:48.507947 22436 catalog_manager.cc:5484] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index -1 to 4, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 07:58:48.509431 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 2b5aafb8ddb24e51867f856884a6ee33 (DEFAULT_TABLE table=rebalancer_extra_table_16 [id=b1544441d1b844aba0eaffbfd1787186]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:48.509717 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2b5aafb8ddb24e51867f856884a6ee33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.510824 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 6c781176e9324a64968f9dcb526c532b (DEFAULT_TABLE table=rebalancer_extra_table_16 [id=b1544441d1b844aba0eaffbfd1787186]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:48.511072 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6c781176e9324a64968f9dcb526c532b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.509431 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 6c781176e9324a64968f9dcb526c532b (DEFAULT_TABLE table=rebalancer_extra_table_16 [id=b1544441d1b844aba0eaffbfd1787186]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:48.512084 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6c781176e9324a64968f9dcb526c532b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.512912 24851 tablet_service.cc:1467] Processing CreateTablet for tablet 2b5aafb8ddb24e51867f856884a6ee33 (DEFAULT_TABLE table=rebalancer_extra_table_16 [id=b1544441d1b844aba0eaffbfd1787186]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:48.513120 24851 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2b5aafb8ddb24e51867f856884a6ee33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.514251 28476 tablet_bootstrap.cc:492] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:48.514223 25246 tablet_service.cc:1467] Processing CreateTablet for tablet e032524f34e04ffea0c001e403e840c9 (DEFAULT_TABLE table=rebalancer_extra_table_16 [id=b1544441d1b844aba0eaffbfd1787186]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:48.514530 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e032524f34e04ffea0c001e403e840c9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.515188 28476 tablet_bootstrap.cc:654] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:48.515173 24850 tablet_service.cc:1467] Processing CreateTablet for tablet e032524f34e04ffea0c001e403e840c9 (DEFAULT_TABLE table=rebalancer_extra_table_16 [id=b1544441d1b844aba0eaffbfd1787186]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:48.515246 22566 tablet_service.cc:1467] Processing CreateTablet for tablet e032524f34e04ffea0c001e403e840c9 (DEFAULT_TABLE table=rebalancer_extra_table_16 [id=b1544441d1b844aba0eaffbfd1787186]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:48.515376 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e032524f34e04ffea0c001e403e840c9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.515502 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e032524f34e04ffea0c001e403e840c9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.516978 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 6c781176e9324a64968f9dcb526c532b (DEFAULT_TABLE table=rebalancer_extra_table_16 [id=b1544441d1b844aba0eaffbfd1787186]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:48.517213 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6c781176e9324a64968f9dcb526c532b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.517827 28479 tablet_bootstrap.cc:492] T e032524f34e04ffea0c001e403e840c9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:48.518733 28479 tablet_bootstrap.cc:654] T e032524f34e04ffea0c001e403e840c9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:48.518854 28476 tablet_bootstrap.cc:492] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:48.518934 28476 ts_tablet_manager.cc:1367] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:48.519199 28480 tablet_bootstrap.cc:492] T 6c781176e9324a64968f9dcb526c532b P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:48.519353 28476 raft_consensus.cc:348] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.519429 24983 tablet_service.cc:1467] Processing CreateTablet for tablet 2b5aafb8ddb24e51867f856884a6ee33 (DEFAULT_TABLE table=rebalancer_extra_table_16 [id=b1544441d1b844aba0eaffbfd1787186]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:48.519512 28476 raft_consensus.cc:374] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:48.519644 28476 raft_consensus.cc:729] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:48.519737 24983 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2b5aafb8ddb24e51867f856884a6ee33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.519882 28476 consensus_queue.cc:260] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.519987 28479 tablet_bootstrap.cc:492] T e032524f34e04ffea0c001e403e840c9 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:48.520071 28479 ts_tablet_manager.cc:1367] T e032524f34e04ffea0c001e403e840c9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:48.520212 28480 tablet_bootstrap.cc:654] T 6c781176e9324a64968f9dcb526c532b P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:48.520427 28479 raft_consensus.cc:348] T e032524f34e04ffea0c001e403e840c9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:48.520520 28479 raft_consensus.cc:374] T e032524f34e04ffea0c001e403e840c9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:48.520547 28479 raft_consensus.cc:729] T e032524f34e04ffea0c001e403e840c9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:48.520808 28476 ts_tablet_manager.cc:1398] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:48.520962 28476 tablet_bootstrap.cc:492] T 6c781176e9324a64968f9dcb526c532b P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:48.520905 28479 consensus_queue.cc:260] T e032524f34e04ffea0c001e403e840c9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:48.521163 28479 ts_tablet_manager.cc:1398] T e032524f34e04ffea0c001e403e840c9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:48.521880 28483 tablet_bootstrap.cc:492] T 2b5aafb8ddb24e51867f856884a6ee33 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:48.521894 28476 tablet_bootstrap.cc:654] T 6c781176e9324a64968f9dcb526c532b P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:48.522084 28480 tablet_bootstrap.cc:492] T 6c781176e9324a64968f9dcb526c532b P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:48.522219 28480 ts_tablet_manager.cc:1367] T 6c781176e9324a64968f9dcb526c532b P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:48.522734 28480 raft_consensus.cc:348] T 6c781176e9324a64968f9dcb526c532b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.522883 28480 raft_consensus.cc:374] T 6c781176e9324a64968f9dcb526c532b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:48.522964 28480 raft_consensus.cc:729] T 6c781176e9324a64968f9dcb526c532b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:48.523036 28476 tablet_bootstrap.cc:492] T 6c781176e9324a64968f9dcb526c532b P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:48.523109 28476 ts_tablet_manager.cc:1367] T 6c781176e9324a64968f9dcb526c532b P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:48.523150 28480 consensus_queue.cc:260] T 6c781176e9324a64968f9dcb526c532b P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.523324 28483 tablet_bootstrap.cc:654] T 2b5aafb8ddb24e51867f856884a6ee33 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:48.523530 28476 raft_consensus.cc:348] T 6c781176e9324a64968f9dcb526c532b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.523615 28476 raft_consensus.cc:374] T 6c781176e9324a64968f9dcb526c532b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:48.523645 28476 raft_consensus.cc:729] T 6c781176e9324a64968f9dcb526c532b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:48.523756 28476 consensus_queue.cc:260] T 6c781176e9324a64968f9dcb526c532b P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.523999 28476 ts_tablet_manager.cc:1398] T 6c781176e9324a64968f9dcb526c532b P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:48.524153 28476 tablet_bootstrap.cc:492] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:48.525040 28476 tablet_bootstrap.cc:654] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:48.525120 28477 tablet_bootstrap.cc:492] T 2b5aafb8ddb24e51867f856884a6ee33 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:48.526042 22716 consensus_queue.cc:237] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:48.526149 28477 tablet_bootstrap.cc:654] T 2b5aafb8ddb24e51867f856884a6ee33 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:48.527783 28476 tablet_bootstrap.cc:492] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:48.527865 28476 ts_tablet_manager.cc:1367] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:48.528067 25003 raft_consensus.cc:1261] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:48.528231 28476 raft_consensus.cc:348] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:48.528316 28476 raft_consensus.cc:374] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:48.528345 28476 raft_consensus.cc:729] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:48.528457 28476 consensus_queue.cc:260] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:48.528688 22845 raft_consensus.cc:1261] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:48.528713 26995 consensus_queue.cc:1035] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:48.529450 28476 ts_tablet_manager.cc:1398] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:48.529821 25134 raft_consensus.cc:1203] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:48.530253 28483 tablet_bootstrap.cc:492] T 2b5aafb8ddb24e51867f856884a6ee33 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:48.530519 28483 ts_tablet_manager.cc:1367] T 2b5aafb8ddb24e51867f856884a6ee33 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.009s user 0.002s sys 0.000s
I20240502 07:58:48.530712 28477 tablet_bootstrap.cc:492] T 2b5aafb8ddb24e51867f856884a6ee33 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:48.530890 28477 ts_tablet_manager.cc:1367] T 2b5aafb8ddb24e51867f856884a6ee33 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:58:48.531016 28483 raft_consensus.cc:348] T 2b5aafb8ddb24e51867f856884a6ee33 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.531183 28483 raft_consensus.cc:374] T 2b5aafb8ddb24e51867f856884a6ee33 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:48.531255 28483 raft_consensus.cc:729] T 2b5aafb8ddb24e51867f856884a6ee33 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:48.531396 28483 consensus_queue.cc:260] T 2b5aafb8ddb24e51867f856884a6ee33 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.531412 27303 consensus_queue.cc:1035] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:48.531541 28477 raft_consensus.cc:348] T 2b5aafb8ddb24e51867f856884a6ee33 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.531764 28477 raft_consensus.cc:374] T 2b5aafb8ddb24e51867f856884a6ee33 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:48.531854 28477 raft_consensus.cc:729] T 2b5aafb8ddb24e51867f856884a6ee33 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:48.532021 28477 consensus_queue.cc:260] T 2b5aafb8ddb24e51867f856884a6ee33 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.532747 28480 ts_tablet_manager.cc:1398] T 6c781176e9324a64968f9dcb526c532b P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.010s user 0.001s sys 0.000s
I20240502 07:58:48.533398 26995 raft_consensus.cc:2932] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.533897 28477 ts_tablet_manager.cc:1398] T 2b5aafb8ddb24e51867f856884a6ee33 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:48.534039 28477 tablet_bootstrap.cc:492] T e032524f34e04ffea0c001e403e840c9 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:48.534773 28483 ts_tablet_manager.cc:1398] T 2b5aafb8ddb24e51867f856884a6ee33 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:48.534888 28477 tablet_bootstrap.cc:654] T e032524f34e04ffea0c001e403e840c9 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:48.535137 22428 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cceaa8887081401dbc9154d265004c7b with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:48.535677 22845 raft_consensus.cc:2932] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.537230 28477 tablet_bootstrap.cc:492] T e032524f34e04ffea0c001e403e840c9 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:48.537344 28477 ts_tablet_manager.cc:1367] T e032524f34e04ffea0c001e403e840c9 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:48.537763 28477 raft_consensus.cc:348] T e032524f34e04ffea0c001e403e840c9 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:48.537988 28477 raft_consensus.cc:374] T e032524f34e04ffea0c001e403e840c9 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:48.538085 28477 raft_consensus.cc:729] T e032524f34e04ffea0c001e403e840c9 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:48.538422 22442 catalog_manager.cc:5484] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.538270 28477 consensus_queue.cc:260] T e032524f34e04ffea0c001e403e840c9 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:48.539426 28477 ts_tablet_manager.cc:1398] T e032524f34e04ffea0c001e403e840c9 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:48.540199 25004 raft_consensus.cc:2932] T cceaa8887081401dbc9154d265004c7b P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.541435 28477 tablet_bootstrap.cc:492] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:48.542033 27582 raft_consensus.cc:1050] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3: attempting to promote NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e to VOTER
I20240502 07:58:48.542008 25670 consensus_queue.cc:1035] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:48.542320 28477 tablet_bootstrap.cc:654] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:48.542403 27582 consensus_queue.cc:237] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:48.543777 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet cceaa8887081401dbc9154d265004c7b with delete_type TABLET_DATA_TOMBSTONED (TS 02a92e1c084b4308b9e855a4cc2c295f not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:48.543987 28131 tablet_replica.cc:330] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:48.544118 28131 raft_consensus.cc:2229] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:48.544275 28131 raft_consensus.cc:2258] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:48.544773 28477 tablet_bootstrap.cc:492] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:48.544816 24871 raft_consensus.cc:1261] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.544857 28477 ts_tablet_manager.cc:1367] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:48.545043 28131 ts_tablet_manager.cc:1875] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:48.545264 27303 consensus_queue.cc:1035] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.545353 28477 raft_consensus.cc:348] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.545441 28477 raft_consensus.cc:374] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:48.545470 28477 raft_consensus.cc:729] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:48.545560 28477 consensus_queue.cc:260] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.545790 25265 raft_consensus.cc:1261] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.546284 28477 ts_tablet_manager.cc:1398] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:48.548760 25670 consensus_queue.cc:1035] T 9f55942f203d455394d2f81f9ae91f0f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:48.549015 26995 consensus_queue.cc:1035] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.549412 25004 raft_consensus.cc:1261] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:48.550156 28131 ts_tablet_manager.cc:1888] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:48.550230 28131 log.cc:1196] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/cceaa8887081401dbc9154d265004c7b
I20240502 07:58:48.550863 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet cceaa8887081401dbc9154d265004c7b (table rebalance_test_table_9 [id=bc838e66a1294569a905266b5619a7d7]) successfully deleted
I20240502 07:58:48.551077 28450 consensus_queue.cc:1035] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:48.561625 26995 raft_consensus.cc:2932] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.562029 25265 raft_consensus.cc:2932] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.564070 24871 raft_consensus.cc:2932] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.564769 22436 catalog_manager.cc:5484] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 2 to 3, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:48.564819 25004 raft_consensus.cc:2932] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.567790 24871 raft_consensus.cc:1203] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 07:58:48.571635 22716 consensus_queue.cc:237] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:48.572516 24871 raft_consensus.cc:1261] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:48.572526 25004 raft_consensus.cc:1261] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:48.572818 28450 consensus_queue.cc:1035] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:48.573078 27303 consensus_queue.cc:1035] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:48.574294 27303 raft_consensus.cc:2932] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.575229 25004 raft_consensus.cc:2932] T c3a7b2e52ba641e993cf1d5dc9724473 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.575686 22428 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c3a7b2e52ba641e993cf1d5dc9724473 with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:48.575999 24871 raft_consensus.cc:2932] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.577798 22436 catalog_manager.cc:5484] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 3 to 4, VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) evicted. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:48.580376 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet c3a7b2e52ba641e993cf1d5dc9724473 with delete_type TABLET_DATA_TOMBSTONED (TS 7eced220cf1642f3873fc385b6191e69 not found in new config with opid_index 4) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:48.580749 28510 tablet_replica.cc:330] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:48.580931 28510 raft_consensus.cc:2229] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:48.581097 28510 raft_consensus.cc:2258] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:48.581641 28510 ts_tablet_manager.cc:1875] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:48.583236 28510 ts_tablet_manager.cc:1888] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.3
I20240502 07:58:48.583297 28510 log.cc:1196] T c3a7b2e52ba641e993cf1d5dc9724473 P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/c3a7b2e52ba641e993cf1d5dc9724473
I20240502 07:58:48.583813 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet c3a7b2e52ba641e993cf1d5dc9724473 (table rebalancer_extra_table_4 [id=61a72ea9962740bd8839987228b47853]) successfully deleted
I20240502 07:58:48.599604 22438 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:48.599714 22438 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:48.601586 28522 ts_tablet_manager.cc:902] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69: Initiating tablet copy from peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:48.601850 22438 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 8e70e35cb8da46c7b8612266d21f7ac9
I20240502 07:58:48.602391 24983 tablet_service.cc:1514] Processing DeleteTablet for tablet 8e70e35cb8da46c7b8612266d21f7ac9 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:48 UTC) from {username='slave'} at 127.0.0.1:42878
I20240502 07:58:48.602419 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet 8e70e35cb8da46c7b8612266d21f7ac9 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:48 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:48.602528 28522 tablet_copy_client.cc:308] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:37451
I20240502 07:58:48.602555 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 8e70e35cb8da46c7b8612266d21f7ac9 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:48 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:48.602589 28131 tablet_replica.cc:330] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:48.602695 28352 tablet_replica.cc:330] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:48.602705 28043 tablet_replica.cc:330] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e: stopping tablet replica
I20240502 07:58:48.602725 28131 raft_consensus.cc:2229] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:48.602818 28352 raft_consensus.cc:2229] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:48.602823 28043 raft_consensus.cc:2229] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:48.602841 28131 raft_consensus.cc:2258] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:48.602922 28352 raft_consensus.cc:2258] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:48.602995 28043 raft_consensus.cc:2258] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:48.603389 22606 tablet_copy_service.cc:140] P 8bea0fe8d8044a2685f92f7a10c9cf85: Received BeginTabletCopySession request for tablet dd815609d7324cc588f02b6cb40cfbfd from peer 7eced220cf1642f3873fc385b6191e69 ({username='slave'} at 127.16.85.136:57807)
I20240502 07:58:48.603467 22606 tablet_copy_service.cc:161] P 8bea0fe8d8044a2685f92f7a10c9cf85: Beginning new tablet copy session on tablet dd815609d7324cc588f02b6cb40cfbfd from peer 7eced220cf1642f3873fc385b6191e69 at {username='slave'} at 127.16.85.136:57807: session id = 7eced220cf1642f3873fc385b6191e69-dd815609d7324cc588f02b6cb40cfbfd
I20240502 07:58:48.603539 28131 ts_tablet_manager.cc:1875] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:48.603739 28043 ts_tablet_manager.cc:1875] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:48.604319 22606 tablet_copy_source_session.cc:205] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:48.605284 28352 ts_tablet_manager.cc:1875] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:48.605581 28043 ts_tablet_manager.cc:1888] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:48.605633 28043 log.cc:1196] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/wals/8e70e35cb8da46c7b8612266d21f7ac9
I20240502 07:58:48.605720 28131 ts_tablet_manager.cc:1888] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:48.605774 28131 log.cc:1196] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/8e70e35cb8da46c7b8612266d21f7ac9
I20240502 07:58:48.605865 28043 ts_tablet_manager.cc:1909] T 8e70e35cb8da46c7b8612266d21f7ac9 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting consensus metadata
I20240502 07:58:48.606009 28131 ts_tablet_manager.cc:1909] T 8e70e35cb8da46c7b8612266d21f7ac9 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:48.606489 22430 catalog_manager.cc:4830] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): tablet 8e70e35cb8da46c7b8612266d21f7ac9 (table rebalancer_dynamic_table_new_name [id=5720ff0cb45c41739920a50502ed7847]) successfully deleted
I20240502 07:58:48.606700 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 8e70e35cb8da46c7b8612266d21f7ac9 (table rebalancer_dynamic_table_new_name [id=5720ff0cb45c41739920a50502ed7847]) successfully deleted
I20240502 07:58:48.607488 28352 ts_tablet_manager.cc:1888] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:48.607563 28352 log.cc:1196] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/8e70e35cb8da46c7b8612266d21f7ac9
I20240502 07:58:48.607805 28352 ts_tablet_manager.cc:1909] T 8e70e35cb8da46c7b8612266d21f7ac9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:48.608347 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 8e70e35cb8da46c7b8612266d21f7ac9 (table rebalancer_dynamic_table_new_name [id=5720ff0cb45c41739920a50502ed7847]) successfully deleted
I20240502 07:58:48.608948 28522 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet dd815609d7324cc588f02b6cb40cfbfd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.611119 28522 tablet_copy_client.cc:783] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:48.611253 28522 tablet_copy_client.cc:647] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:48.612334 28522 tablet_copy_client.cc:523] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:48.613415 28522 tablet_bootstrap.cc:492] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:48.626353 27582 raft_consensus.cc:1050] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3: attempting to promote NON_VOTER 7eced220cf1642f3873fc385b6191e69 to VOTER
I20240502 07:58:48.626690 27582 consensus_queue.cc:237] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:48.627914 22584 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:48.628017 25004 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:48.628140 25265 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:48.628293 26995 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:48.629179 26995 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:48.629341 28470 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:48.629537 27339 raft_consensus.cc:482] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:48.629662 27339 raft_consensus.cc:504] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.630013 27339 leader_election.cc:288] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:48.630604 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6c781176e9324a64968f9dcb526c532b" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:48.630739 25134 raft_consensus.cc:2454] T 6c781176e9324a64968f9dcb526c532b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:48.631060 24808 leader_election.cc:302] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:48.631228 27339 raft_consensus.cc:2781] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:48.631274 27339 raft_consensus.cc:482] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:48.631307 27339 raft_consensus.cc:3037] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:48.631652 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6c781176e9324a64968f9dcb526c532b" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:48.631753 25265 raft_consensus.cc:2454] T 6c781176e9324a64968f9dcb526c532b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:48.632283 27339 raft_consensus.cc:504] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.632573 27339 leader_election.cc:288] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:48.633054 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6c781176e9324a64968f9dcb526c532b" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:48.633147 25134 raft_consensus.cc:3037] T 6c781176e9324a64968f9dcb526c532b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:48.633994 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6c781176e9324a64968f9dcb526c532b" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:48.634090 25265 raft_consensus.cc:3037] T 6c781176e9324a64968f9dcb526c532b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:48.634138 25134 raft_consensus.cc:2454] T 6c781176e9324a64968f9dcb526c532b P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:48.634464 24808 leader_election.cc:302] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:48.634624 27339 raft_consensus.cc:2781] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:48.634693 27339 raft_consensus.cc:686] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:58:48.634824 27339 consensus_queue.cc:237] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.634943 25265 raft_consensus.cc:2454] T 6c781176e9324a64968f9dcb526c532b P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:48.635202 28470 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:48.637269 22438 catalog_manager.cc:5484] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:48.637722 22584 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:48.640640 22438 catalog_manager.cc:5484] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:48.640851 28522 tablet_bootstrap.cc:492] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:48.641095 28522 tablet_bootstrap.cc:492] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69: Bootstrap complete.
I20240502 07:58:48.641248 28522 ts_tablet_manager.cc:1367] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.028s user 0.022s sys 0.004s
I20240502 07:58:48.641496 25004 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:48.641678 28522 raft_consensus.cc:348] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:48.641808 28522 raft_consensus.cc:729] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: LEARNER
I20240502 07:58:48.641916 28522 consensus_queue.cc:260] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:48.642263 28522 ts_tablet_manager.cc:1398] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:48.642745 22606 tablet_copy_service.cc:342] P 8bea0fe8d8044a2685f92f7a10c9cf85: Request end of tablet copy session 7eced220cf1642f3873fc385b6191e69-dd815609d7324cc588f02b6cb40cfbfd received from {username='slave'} at 127.16.85.136:57807
I20240502 07:58:48.642814 22606 tablet_copy_service.cc:433] P 8bea0fe8d8044a2685f92f7a10c9cf85: ending tablet copy session 7eced220cf1642f3873fc385b6191e69-dd815609d7324cc588f02b6cb40cfbfd on tablet dd815609d7324cc588f02b6cb40cfbfd with peer 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:48.643097 25265 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:48.644318 22716 consensus_queue.cc:237] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:48.646116 25265 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 07:58:48.646138 25004 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 07:58:48.646795 27582 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:48.647178 28450 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:48.648550 27582 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:48.648945 25265 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:48.649786 25004 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:48.650084 22428 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e25ba5cc77334a3595a4575fa98476f0 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:48.651379 22438 catalog_manager.cc:5484] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: config changed from index 8 to 9, VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) evicted. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:48.653460 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet e25ba5cc77334a3595a4575fa98476f0 with delete_type TABLET_DATA_TOMBSTONED (TS 8bea0fe8d8044a2685f92f7a10c9cf85 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:48.653661 28352 tablet_replica.cc:330] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:48.653805 28352 raft_consensus.cc:2229] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:48.654009 28352 raft_consensus.cc:2258] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:48.654778 28352 ts_tablet_manager.cc:1875] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:48.656402 28352 ts_tablet_manager.cc:1888] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 07:58:48.656472 28352 log.cc:1196] T e25ba5cc77334a3595a4575fa98476f0 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/e25ba5cc77334a3595a4575fa98476f0
I20240502 07:58:48.656973 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet e25ba5cc77334a3595a4575fa98476f0 (table rebalance_test_table_11 [id=0c1156cac15d48028ebf5c07c67ea832]) successfully deleted
I20240502 07:58:48.658434 28558 ts_tablet_manager.cc:902] T da39029564d042f6acfc23f99fb63dfd P 74f9aa91c2c448f980b512fe2e8d672e: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:48.658954 28558 tablet_copy_client.cc:308] T da39029564d042f6acfc23f99fb63dfd P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33387
I20240502 07:58:48.659399 22737 tablet_copy_service.cc:140] P 725969d490a04fdfb3ba890c3f5096e3: Received BeginTabletCopySession request for tablet da39029564d042f6acfc23f99fb63dfd from peer 74f9aa91c2c448f980b512fe2e8d672e ({username='slave'} at 127.16.85.134:53317)
I20240502 07:58:48.659490 22737 tablet_copy_service.cc:161] P 725969d490a04fdfb3ba890c3f5096e3: Beginning new tablet copy session on tablet da39029564d042f6acfc23f99fb63dfd from peer 74f9aa91c2c448f980b512fe2e8d672e at {username='slave'} at 127.16.85.134:53317: session id = 74f9aa91c2c448f980b512fe2e8d672e-da39029564d042f6acfc23f99fb63dfd
I20240502 07:58:48.660413 22737 tablet_copy_source_session.cc:205] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:48.661285 28558 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet da39029564d042f6acfc23f99fb63dfd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.663523 28558 tablet_copy_client.cc:783] T da39029564d042f6acfc23f99fb63dfd P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:48.663743 28558 tablet_copy_client.cc:647] T da39029564d042f6acfc23f99fb63dfd P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:48.664759 28558 tablet_copy_client.cc:523] T da39029564d042f6acfc23f99fb63dfd P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:48.665755 28558 tablet_bootstrap.cc:492] T da39029564d042f6acfc23f99fb63dfd P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:48.667455 22845 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "66ac69e448574c7ea5101014e0dd425d"
dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:50406
I20240502 07:58:48.667618 22845 raft_consensus.cc:595] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:48.669423 28558 tablet_bootstrap.cc:492] T da39029564d042f6acfc23f99fb63dfd P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:48.669570 28574 ts_tablet_manager.cc:902] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f: Initiating tablet copy from peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:48.669719 28558 tablet_bootstrap.cc:492] T da39029564d042f6acfc23f99fb63dfd P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap complete.
I20240502 07:58:48.669876 28558 ts_tablet_manager.cc:1367] T da39029564d042f6acfc23f99fb63dfd P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.004s user 0.004s sys 0.000s
I20240502 07:58:48.670024 28574 tablet_copy_client.cc:308] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.135:41205
I20240502 07:58:48.670481 28558 raft_consensus.cc:348] T da39029564d042f6acfc23f99fb63dfd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:48.670542 25155 tablet_copy_service.cc:140] P 4654231dfe994c1a9d2e37d9c3d75699: Received BeginTabletCopySession request for tablet 5487e66cb4d8416094a1321cb989f018 from peer 02a92e1c084b4308b9e855a4cc2c295f ({username='slave'} at 127.16.85.132:50937)
I20240502 07:58:48.670676 25155 tablet_copy_service.cc:161] P 4654231dfe994c1a9d2e37d9c3d75699: Beginning new tablet copy session on tablet 5487e66cb4d8416094a1321cb989f018 from peer 02a92e1c084b4308b9e855a4cc2c295f at {username='slave'} at 127.16.85.132:50937: session id = 02a92e1c084b4308b9e855a4cc2c295f-5487e66cb4d8416094a1321cb989f018
I20240502 07:58:48.670679 28558 raft_consensus.cc:729] T da39029564d042f6acfc23f99fb63dfd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: LEARNER
I20240502 07:58:48.670804 28558 consensus_queue.cc:260] T da39029564d042f6acfc23f99fb63dfd P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:48.671530 28558 ts_tablet_manager.cc:1398] T da39029564d042f6acfc23f99fb63dfd P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:48.671891 25155 tablet_copy_source_session.cc:205] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:48.671919 22737 tablet_copy_service.cc:342] P 725969d490a04fdfb3ba890c3f5096e3: Request end of tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-da39029564d042f6acfc23f99fb63dfd received from {username='slave'} at 127.16.85.134:53317
I20240502 07:58:48.671970 22737 tablet_copy_service.cc:433] P 725969d490a04fdfb3ba890c3f5096e3: ending tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-da39029564d042f6acfc23f99fb63dfd on tablet da39029564d042f6acfc23f99fb63dfd with peer 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:48.673166 28574 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5487e66cb4d8416094a1321cb989f018. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.675510 28574 tablet_copy_client.cc:783] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:48.675680 28574 tablet_copy_client.cc:647] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:48.677515 28574 tablet_copy_client.cc:523] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:48.678838 28574 tablet_bootstrap.cc:492] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:48.682596 28574 tablet_bootstrap.cc:492] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:48.682802 28574 tablet_bootstrap.cc:492] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap complete.
I20240502 07:58:48.682951 28574 ts_tablet_manager.cc:1367] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:48.683272 28574 raft_consensus.cc:348] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 07:58:48.683360 28574 raft_consensus.cc:729] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: LEARNER
I20240502 07:58:48.683477 28574 consensus_queue.cc:260] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 07:58:48.683774 28574 ts_tablet_manager.cc:1398] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:48.684306 25155 tablet_copy_service.cc:342] P 4654231dfe994c1a9d2e37d9c3d75699: Request end of tablet copy session 02a92e1c084b4308b9e855a4cc2c295f-5487e66cb4d8416094a1321cb989f018 received from {username='slave'} at 127.16.85.132:50937
I20240502 07:58:48.684379 25155 tablet_copy_service.cc:433] P 4654231dfe994c1a9d2e37d9c3d75699: ending tablet copy session 02a92e1c084b4308b9e855a4cc2c295f-5487e66cb4d8416094a1321cb989f018 on tablet 5487e66cb4d8416094a1321cb989f018 with peer 02a92e1c084b4308b9e855a4cc2c295f
I20240502 07:58:48.691107 27495 raft_consensus.cc:482] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:48.691193 27495 raft_consensus.cc:504] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:48.691524 27495 leader_election.cc:288] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:48.691829 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e032524f34e04ffea0c001e403e840c9" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:48.691918 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e032524f34e04ffea0c001e403e840c9" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:48.691967 24871 raft_consensus.cc:2454] T e032524f34e04ffea0c001e403e840c9 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:48.692042 22584 raft_consensus.cc:2454] T e032524f34e04ffea0c001e403e840c9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:48.692268 25199 leader_election.cc:302] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:48.692494 27495 raft_consensus.cc:2781] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:48.692551 27495 raft_consensus.cc:482] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:48.692586 27495 raft_consensus.cc:3037] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:48.693564 27495 raft_consensus.cc:504] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:48.693895 27495 leader_election.cc:288] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:48.694489 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e032524f34e04ffea0c001e403e840c9" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:48.694593 22584 raft_consensus.cc:3037] T e032524f34e04ffea0c001e403e840c9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:48.695456 22584 raft_consensus.cc:2454] T e032524f34e04ffea0c001e403e840c9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:48.695757 25202 leader_election.cc:302] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7eced220cf1642f3873fc385b6191e69, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:48.695811 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e032524f34e04ffea0c001e403e840c9" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:48.695919 24871 raft_consensus.cc:3037] T e032524f34e04ffea0c001e403e840c9 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:48.695914 27495 raft_consensus.cc:2781] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:48.695992 27495 raft_consensus.cc:686] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:48.696136 27495 consensus_queue.cc:237] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:48.696796 24871 raft_consensus.cc:2454] T e032524f34e04ffea0c001e403e840c9 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:48.698513 22444 catalog_manager.cc:5484] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:48.704087 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:48.708768 25115 tablet_service.cc:1467] Processing CreateTablet for tablet cf2982b56cb24afca4ada89a8cbd6645 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=3bcedb75536c4dc9b765c1f785166da1]), partition=
I20240502 07:58:48.708798 22697 tablet_service.cc:1467] Processing CreateTablet for tablet cf2982b56cb24afca4ada89a8cbd6645 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=3bcedb75536c4dc9b765c1f785166da1]), partition=
I20240502 07:58:48.709007 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf2982b56cb24afca4ada89a8cbd6645. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.709007 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf2982b56cb24afca4ada89a8cbd6645. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.711040 28480 tablet_bootstrap.cc:492] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:48.711093 28611 tablet_bootstrap.cc:492] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:48.711920 28611 tablet_bootstrap.cc:654] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:48.711942 28480 tablet_bootstrap.cc:654] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:48.713023 24983 tablet_service.cc:1467] Processing CreateTablet for tablet cf2982b56cb24afca4ada89a8cbd6645 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=3bcedb75536c4dc9b765c1f785166da1]), partition=
I20240502 07:58:48.713228 24983 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf2982b56cb24afca4ada89a8cbd6645. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.713225 22845 consensus_queue.cc:237] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:48.714563 22716 raft_consensus.cc:1261] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:48.714833 28611 tablet_bootstrap.cc:492] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:48.714900 28611 ts_tablet_manager.cc:1367] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:48.715252 28611 raft_consensus.cc:348] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:48.715330 28611 raft_consensus.cc:374] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:48.715345 28483 tablet_bootstrap.cc:492] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:48.715355 28611 raft_consensus.cc:729] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:48.715435 28611 consensus_queue.cc:260] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:48.715653 28611 ts_tablet_manager.cc:1398] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:48.716096 28483 tablet_bootstrap.cc:654] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:48.716883 25532 consensus_queue.cc:1035] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:48.717206 22978 raft_consensus.cc:1261] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:48.717743 25532 consensus_queue.cc:1035] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 07:58:48.717886 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: bd1c2abfe2a3416bb10d6c14d3fed46a. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:48.719564 28480 tablet_bootstrap.cc:492] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:48.719635 28480 ts_tablet_manager.cc:1367] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.009s user 0.001s sys 0.000s
I20240502 07:58:48.720031 28480 raft_consensus.cc:348] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:48.720111 28480 raft_consensus.cc:374] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:48.720136 28480 raft_consensus.cc:729] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:48.720247 28480 consensus_queue.cc:260] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:48.720489 28480 ts_tablet_manager.cc:1398] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:48.721177 28483 tablet_bootstrap.cc:492] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:48.721240 28483 ts_tablet_manager.cc:1367] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:58:48.721572 28483 raft_consensus.cc:348] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:48.721648 28483 raft_consensus.cc:374] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:48.721671 28483 raft_consensus.cc:729] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:48.721765 28483 consensus_queue.cc:260] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:48.722029 28483 ts_tablet_manager.cc:1398] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:48.723899 25532 raft_consensus.cc:2932] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:48.727506 22978 raft_consensus.cc:2932] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:48.729264 22438 catalog_manager.cc:5484] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:48.734267 22716 raft_consensus.cc:2932] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:48.751719 22978 consensus_queue.cc:237] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:48.753082 22845 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 07:58:48.753217 22915 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): Couldn't send request to peer 4654231dfe994c1a9d2e37d9c3d75699. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f1fa312fe8e44d23970c5e3ba98c2c58. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:48.753667 25004 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 07:58:48.754019 27333 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:48.754179 27310 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:48.755825 27333 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:48.756661 22845 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:48.758589 25004 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:48.758811 22436 catalog_manager.cc:5484] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 6 to 7, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:48.762322 24714 raft_consensus.cc:1050] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85: attempting to promote NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e to VOTER
I20240502 07:58:48.762756 24714 consensus_queue.cc:237] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:48.764129 22716 raft_consensus.cc:1261] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.764750 28188 consensus_queue.cc:1035] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.765520 25004 raft_consensus.cc:1261] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.766114 28188 consensus_queue.cc:1035] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.766168 22845 raft_consensus.cc:1261] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.766727 27136 consensus_queue.cc:1035] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.768041 27136 raft_consensus.cc:2932] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.768628 22845 raft_consensus.cc:2932] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.769479 25004 raft_consensus.cc:2932] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.771019 22435 catalog_manager.cc:5484] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.771287 22716 raft_consensus.cc:2932] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.776149 22584 consensus_queue.cc:237] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:48.776486 22586 consensus_queue.cc:237] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
I20240502 07:58:48.777546 25004 raft_consensus.cc:1261] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:48.777930 24871 raft_consensus.cc:1261] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 07:58:48.777947 22716 raft_consensus.cc:1261] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 07:58:48.778069 22845 raft_consensus.cc:1261] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:48.778324 28188 consensus_queue.cc:1035] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:48.778429 27023 consensus_queue.cc:1035] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
W20240502 07:58:48.778517 22521 consensus_peers.cc:485] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Couldn't send request to peer 7eced220cf1642f3873fc385b6191e69. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9e5fe86b44744df9b2e4d430fdec716f. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:48.778672 28188 consensus_queue.cc:1035] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:48.778916 27136 consensus_queue.cc:1035] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:48.780289 27023 raft_consensus.cc:2932] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.780634 25004 raft_consensus.cc:2932] T a5374313bbba4b10904e0ea18851c07d P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.780925 24714 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:48.781327 24871 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:48.782464 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a5374313bbba4b10904e0ea18851c07d with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:48.782747 22845 raft_consensus.cc:2932] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:48.783418 27488 raft_consensus.cc:482] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:48.783529 27488 raft_consensus.cc:504] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:48.783967 27488 leader_election.cc:288] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:48.783985 22436 catalog_manager.cc:5484] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 5 to 6, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:48.784332 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf2982b56cb24afca4ada89a8cbd6645" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:48.784441 22717 raft_consensus.cc:2454] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:48.784507 25004 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf2982b56cb24afca4ada89a8cbd6645" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e" is_pre_election: true
I20240502 07:58:48.784649 25004 raft_consensus.cc:2454] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:48.784554 22435 catalog_manager.cc:5484] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537 reported cstate change: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:48.784936 25068 leader_election.cc:302] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:48.785159 27488 raft_consensus.cc:2781] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:48.785207 27488 raft_consensus.cc:482] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:48.785244 27488 raft_consensus.cc:3037] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:48.786087 27488 raft_consensus.cc:504] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:48.786198 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet a5374313bbba4b10904e0ea18851c07d with delete_type TABLET_DATA_TOMBSTONED (TS 725969d490a04fdfb3ba890c3f5096e3 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:48.786394 27488 leader_election.cc:288] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:48.786520 28353 tablet_replica.cc:330] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:48.786674 28353 raft_consensus.cc:2229] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:48.786828 28353 raft_consensus.cc:2258] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:48.786965 25004 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf2982b56cb24afca4ada89a8cbd6645" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e"
I20240502 07:58:48.787053 25004 raft_consensus.cc:3037] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:48.787117 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf2982b56cb24afca4ada89a8cbd6645" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:48.787221 22716 raft_consensus.cc:3037] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:48.787890 25004 raft_consensus.cc:2454] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:48.788103 22716 raft_consensus.cc:2454] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:48.788131 25070 leader_election.cc:302] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:48.788268 27488 raft_consensus.cc:2781] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:48.788399 27488 raft_consensus.cc:686] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:58:48.788519 27488 consensus_queue.cc:237] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:48.788913 22716 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:48.790148 28353 ts_tablet_manager.cc:1875] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:48.790467 22444 catalog_manager.cc:5484] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:48.792153 28353 ts_tablet_manager.cc:1888] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:48.792197 28353 log.cc:1196] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/a5374313bbba4b10904e0ea18851c07d
I20240502 07:58:48.792634 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet a5374313bbba4b10904e0ea18851c07d (table rebalance_test_table_5 [id=cd54bf3678754d7291af91c25c402e8d]) successfully deleted
I20240502 07:58:48.797739 28648 ts_tablet_manager.cc:902] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:48.799125 28648 tablet_copy_client.cc:308] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:48.799572 22999 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet ca85b40df34d4e1c91a55d7dd69e1494 from peer 6d02017318a2461796d431962fe6a537 ({username='slave'} at 127.16.85.133:38655)
I20240502 07:58:48.799659 22999 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet ca85b40df34d4e1c91a55d7dd69e1494 from peer 6d02017318a2461796d431962fe6a537 at {username='slave'} at 127.16.85.133:38655: session id = 6d02017318a2461796d431962fe6a537-ca85b40df34d4e1c91a55d7dd69e1494
I20240502 07:58:48.800415 22999 tablet_copy_source_session.cc:205] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:48.801239 28648 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ca85b40df34d4e1c91a55d7dd69e1494. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.803287 28648 tablet_copy_client.cc:783] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:48.803458 28648 tablet_copy_client.cc:647] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:48.804550 28648 tablet_copy_client.cc:523] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:48.805720 28648 tablet_bootstrap.cc:492] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:48.828547 28648 tablet_bootstrap.cc:492] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:48.828898 28648 tablet_bootstrap.cc:492] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537: Bootstrap complete.
I20240502 07:58:48.829089 28648 ts_tablet_manager.cc:1367] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.023s user 0.019s sys 0.004s
I20240502 07:58:48.829542 28648 raft_consensus.cc:348] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:48.829731 28648 raft_consensus.cc:729] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: LEARNER
I20240502 07:58:48.829846 28648 consensus_queue.cc:260] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:48.830121 28648 ts_tablet_manager.cc:1398] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:48.830559 22999 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 6d02017318a2461796d431962fe6a537-ca85b40df34d4e1c91a55d7dd69e1494 received from {username='slave'} at 127.16.85.133:38655
I20240502 07:58:48.830636 22999 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 6d02017318a2461796d431962fe6a537-ca85b40df34d4e1c91a55d7dd69e1494 on tablet ca85b40df34d4e1c91a55d7dd69e1494 with peer 6d02017318a2461796d431962fe6a537
I20240502 07:58:48.852795 22845 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "66ac69e448574c7ea5101014e0dd425d"
dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:59572
I20240502 07:58:48.852931 22845 raft_consensus.cc:595] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:48.859448 26284 raft_consensus.cc:979] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88: : Instructing follower 8bea0fe8d8044a2685f92f7a10c9cf85 to start an election
I20240502 07:58:48.859556 26284 raft_consensus.cc:1067] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Signalling peer 8bea0fe8d8044a2685f92f7a10c9cf85 to start an election
I20240502 07:58:48.860090 22586 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "66ac69e448574c7ea5101014e0dd425d"
dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
from {username='slave'} at 127.16.85.131:36129
I20240502 07:58:48.860251 22586 raft_consensus.cc:482] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 07:58:48.860313 22586 raft_consensus.cc:3037] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:48.861450 22586 raft_consensus.cc:504] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: false } }
I20240502 07:58:48.861858 22586 leader_election.cc:288] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 2 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:48.862213 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "66ac69e448574c7ea5101014e0dd425d" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 2 candidate_status { last_received { term: 1 index: 3 } } ignore_live_leader: true dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:48.862329 25265 raft_consensus.cc:3037] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:48.862347 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "66ac69e448574c7ea5101014e0dd425d" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 2 candidate_status { last_received { term: 1 index: 3 } } ignore_live_leader: true dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:48.862457 22845 raft_consensus.cc:3032] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 07:58:48.862505 22845 raft_consensus.cc:729] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:48.862675 22845 consensus_queue.cc:260] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: false } }
I20240502 07:58:48.862893 22845 raft_consensus.cc:3037] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:48.863335 25265 raft_consensus.cc:2454] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 2.
I20240502 07:58:48.863930 22845 raft_consensus.cc:2454] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 2.
I20240502 07:58:48.863915 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "66ac69e448574c7ea5101014e0dd425d" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 2 candidate_status { last_received { term: 1 index: 3 } } ignore_live_leader: true dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:48.864033 25134 raft_consensus.cc:3037] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:48.864185 28717 ts_tablet_manager.cc:902] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3: Initiating tablet copy from peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:48.864233 22522 leader_election.cc:302] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 7eced220cf1642f3873fc385b6191e69, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:48.864418 28636 raft_consensus.cc:2781] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:48.864498 28636 raft_consensus.cc:686] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 2 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:48.864717 28717 tablet_copy_client.cc:308] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.134:34857
I20240502 07:58:48.864693 28636 consensus_queue.cc:237] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: false } }
I20240502 07:58:48.865079 25134 raft_consensus.cc:2454] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 2.
I20240502 07:58:48.865165 25024 tablet_copy_service.cc:140] P 74f9aa91c2c448f980b512fe2e8d672e: Received BeginTabletCopySession request for tablet 90acd69ebaba4fb2a67e16172e8d7aa5 from peer 725969d490a04fdfb3ba890c3f5096e3 ({username='slave'} at 127.16.85.130:59591)
I20240502 07:58:48.865258 25024 tablet_copy_service.cc:161] P 74f9aa91c2c448f980b512fe2e8d672e: Beginning new tablet copy session on tablet 90acd69ebaba4fb2a67e16172e8d7aa5 from peer 725969d490a04fdfb3ba890c3f5096e3 at {username='slave'} at 127.16.85.130:59591: session id = 725969d490a04fdfb3ba890c3f5096e3-90acd69ebaba4fb2a67e16172e8d7aa5
I20240502 07:58:48.866189 25024 tablet_copy_source_session.cc:205] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 74f9aa91c2c448f980b512fe2e8d672e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:48.867081 22444 catalog_manager.cc:5484] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 1 to 2, leader changed from 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 2 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:48.868301 28717 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 90acd69ebaba4fb2a67e16172e8d7aa5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:48.870520 28717 tablet_copy_client.cc:783] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:48.870744 28717 tablet_copy_client.cc:647] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:48.871747 28717 tablet_copy_client.cc:523] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:48.872902 28717 tablet_bootstrap.cc:492] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:48.876363 28717 tablet_bootstrap.cc:492] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:48.876569 28717 tablet_bootstrap.cc:492] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap complete.
I20240502 07:58:48.876737 28717 ts_tablet_manager.cc:1367] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.001s
I20240502 07:58:48.877131 28717 raft_consensus.cc:348] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 07:58:48.877249 28717 raft_consensus.cc:729] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: LEARNER
I20240502 07:58:48.877338 28717 consensus_queue.cc:260] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 07:58:48.879664 28717 ts_tablet_manager.cc:1398] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.003s user 0.000s sys 0.001s
I20240502 07:58:48.880038 25024 tablet_copy_service.cc:342] P 74f9aa91c2c448f980b512fe2e8d672e: Request end of tablet copy session 725969d490a04fdfb3ba890c3f5096e3-90acd69ebaba4fb2a67e16172e8d7aa5 received from {username='slave'} at 127.16.85.130:59591
I20240502 07:58:48.880108 25024 tablet_copy_service.cc:433] P 74f9aa91c2c448f980b512fe2e8d672e: ending tablet copy session 725969d490a04fdfb3ba890c3f5096e3-90acd69ebaba4fb2a67e16172e8d7aa5 on tablet 90acd69ebaba4fb2a67e16172e8d7aa5 with peer 725969d490a04fdfb3ba890c3f5096e3
I20240502 07:58:48.885893 28205 raft_consensus.cc:482] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:48.885994 28205 raft_consensus.cc:504] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.886363 28205 leader_election.cc:288] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:48.886642 25004 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2b5aafb8ddb24e51867f856884a6ee33" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e" is_pre_election: true
I20240502 07:58:48.886759 25004 raft_consensus.cc:2454] T 2b5aafb8ddb24e51867f856884a6ee33 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:48.886790 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2b5aafb8ddb24e51867f856884a6ee33" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:48.886912 24871 raft_consensus.cc:2454] T 2b5aafb8ddb24e51867f856884a6ee33 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:48.886886 22978 consensus_queue.cc:237] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:48.887182 25199 leader_election.cc:302] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:48.887332 28205 raft_consensus.cc:2781] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:48.887378 28205 raft_consensus.cc:482] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:48.887411 28205 raft_consensus.cc:3037] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:48.888393 28205 raft_consensus.cc:504] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.888469 25004 raft_consensus.cc:1261] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 07:58:48.888737 28205 leader_election.cc:288] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:48.888861 28071 consensus_queue.cc:1035] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:48.888980 22716 raft_consensus.cc:1261] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 07:58:48.889243 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2b5aafb8ddb24e51867f856884a6ee33" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:48.889334 24871 raft_consensus.cc:3037] T 2b5aafb8ddb24e51867f856884a6ee33 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:48.889322 25004 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2b5aafb8ddb24e51867f856884a6ee33" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e"
I20240502 07:58:48.889406 25004 raft_consensus.cc:3037] T 2b5aafb8ddb24e51867f856884a6ee33 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:48.890259 24871 raft_consensus.cc:2454] T 2b5aafb8ddb24e51867f856884a6ee33 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:48.890298 25004 raft_consensus.cc:2454] T 2b5aafb8ddb24e51867f856884a6ee33 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:48.890522 25199 leader_election.cc:302] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:48.890672 27002 raft_consensus.cc:2781] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:48.890770 27002 raft_consensus.cc:686] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:48.890902 27002 consensus_queue.cc:237] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:48.891639 28071 consensus_queue.cc:1035] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:48.892814 28071 raft_consensus.cc:2932] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:48.893261 22444 catalog_manager.cc:5484] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:48.893528 22716 raft_consensus.cc:2932] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:48.895426 22435 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
W20240502 07:58:48.895562 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Couldn't send request to peer 7eced220cf1642f3873fc385b6191e69. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a8dcc36740be4ca3ac1d43591f7549bd. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:48.895896 25003 raft_consensus.cc:2932] T a8dcc36740be4ca3ac1d43591f7549bd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:48.897536 22438 catalog_manager.cc:5484] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:48.902614 25003 raft_consensus.cc:1261] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:48.902647 22716 raft_consensus.cc:1261] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:48.902941 27488 consensus_queue.cc:1035] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:48.903072 28643 consensus_queue.cc:1035] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:48.905449 22436 catalog_manager.cc:5576] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857) does not have the latest schema for tablet cf2982b56cb24afca4ada89a8cbd6645 (table rebalancer_dynamic_table [id=3bcedb75536c4dc9b765c1f785166da1]). Expected version 1 got 0
I20240502 07:58:48.905722 22438 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet cf2982b56cb24afca4ada89a8cbd6645 (table rebalancer_dynamic_table [id=3bcedb75536c4dc9b765c1f785166da1]). Expected version 1 got 0
I20240502 07:58:48.907124 22438 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet cf2982b56cb24afca4ada89a8cbd6645 (table rebalancer_dynamic_table [id=3bcedb75536c4dc9b765c1f785166da1]). Expected version 1 got 0
I20240502 07:58:48.907179 28752 tablet.cc:1722] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:48.907456 28752 tablet.cc:1620] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:58:48.910467 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=3bcedb75536c4dc9b765c1f785166da1] alter complete (version 1)
I20240502 07:58:48.920889 27582 raft_consensus.cc:1050] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3: attempting to promote NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 to VOTER
I20240502 07:58:48.921236 27582 consensus_queue.cc:237] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:48.922582 22978 raft_consensus.cc:1261] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.922930 22845 raft_consensus.cc:1261] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.923230 25134 raft_consensus.cc:1261] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:48.923264 28450 consensus_queue.cc:1035] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.924464 28450 consensus_queue.cc:1035] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.924794 28751 tablet.cc:1722] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:48.925093 28751 tablet.cc:1620] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:48.925596 27303 consensus_queue.cc:1035] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:48.927364 27303 raft_consensus.cc:2932] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.931340 28320 tablet.cc:1722] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:48.931661 28320 tablet.cc:1620] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e: MemRowSet was empty: no flush needed.
I20240502 07:58:48.931967 22436 catalog_manager.cc:5484] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:48.933770 22438 catalog_manager.cc:5576] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857) does not have the latest schema for tablet cf2982b56cb24afca4ada89a8cbd6645 (table rebalancer_dynamic_table [id=3bcedb75536c4dc9b765c1f785166da1]). Expected version 1 got 0
I20240502 07:58:48.935045 22978 raft_consensus.cc:2932] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.936821 22845 raft_consensus.cc:2932] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.939296 25134 raft_consensus.cc:2932] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.940764 22716 consensus_queue.cc:237] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:48.941823 22845 raft_consensus.cc:1261] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:48.942656 27303 consensus_queue.cc:1035] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:48.944051 25134 raft_consensus.cc:1261] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:48.944710 28450 consensus_queue.cc:1035] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:48.945021 28450 raft_consensus.cc:2932] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.946600 22428 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 55fb5c7475b14e0eaced5b8de1792641 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:48.946978 22845 raft_consensus.cc:2932] T 55fb5c7475b14e0eaced5b8de1792641 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.949451 22438 catalog_manager.cc:5484] T 55fb5c7475b14e0eaced5b8de1792641 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:48.952347 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet 55fb5c7475b14e0eaced5b8de1792641 with delete_type TABLET_DATA_TOMBSTONED (TS 02a92e1c084b4308b9e855a4cc2c295f not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:48.952558 28131 tablet_replica.cc:330] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:48.952821 28131 raft_consensus.cc:2229] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:48.952963 28131 raft_consensus.cc:2258] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:48.953349 25134 raft_consensus.cc:2932] T 55fb5c7475b14e0eaced5b8de1792641 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:48.953655 28131 ts_tablet_manager.cc:1875] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:48.955775 28131 ts_tablet_manager.cc:1888] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:48.955832 28131 log.cc:1196] T 55fb5c7475b14e0eaced5b8de1792641 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/55fb5c7475b14e0eaced5b8de1792641
I20240502 07:58:48.956681 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 55fb5c7475b14e0eaced5b8de1792641 (table rebalance_test_table_5 [id=cd54bf3678754d7291af91c25c402e8d]) successfully deleted
I20240502 07:58:48.987663 28188 raft_consensus.cc:1050] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85: attempting to promote NON_VOTER 6d02017318a2461796d431962fe6a537 to VOTER
I20240502 07:58:48.987990 28188 consensus_queue.cc:237] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:48.989441 22716 raft_consensus.cc:1261] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:48.989771 24871 raft_consensus.cc:1261] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:48.989887 28188 consensus_queue.cc:1035] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:48.990254 28188 consensus_queue.cc:1035] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:48.990321 25265 raft_consensus.cc:1261] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:48.990931 28188 consensus_queue.cc:1035] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:48.992558 28188 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.993207 25265 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.995711 22716 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.995832 22444 catalog_manager.cc:5484] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69 reported cstate change: config changed from index 7 to 8, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.996186 24871 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:48.999490 22586 consensus_queue.cc:237] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:49.000633 25265 raft_consensus.cc:1261] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 07:58:49.001650 27023 consensus_queue.cc:1035] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:49.001847 24871 raft_consensus.cc:1261] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 07:58:49.002120 28636 consensus_queue.cc:1035] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:49.003738 28636 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:49.004055 25265 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:49.005156 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c53e8b3bd774466dbafe4994472975bc with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:49.005717 24871 raft_consensus.cc:2932] T c53e8b3bd774466dbafe4994472975bc P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:49.006515 22444 catalog_manager.cc:5484] T c53e8b3bd774466dbafe4994472975bc P 7eced220cf1642f3873fc385b6191e69 reported cstate change: config changed from index 8 to 9, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:49.008610 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet c53e8b3bd774466dbafe4994472975bc with delete_type TABLET_DATA_TOMBSTONED (TS 725969d490a04fdfb3ba890c3f5096e3 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:49.008841 28353 tablet_replica.cc:330] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:49.008980 28353 raft_consensus.cc:2229] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:49.009141 28353 raft_consensus.cc:2258] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:49.010089 28353 ts_tablet_manager.cc:1875] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:49.011955 28353 ts_tablet_manager.cc:1888] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 07:58:49.012014 28353 log.cc:1196] T c53e8b3bd774466dbafe4994472975bc P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/c53e8b3bd774466dbafe4994472975bc
I20240502 07:58:49.012428 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet c53e8b3bd774466dbafe4994472975bc (table rebalance_test_table_13 [id=2882412f9f504bfc93c45c294c9ace87]) successfully deleted
I20240502 07:58:49.014137 22438 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:49.019147 28751 tablet.cc:1722] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:49.019464 28751 tablet.cc:1620] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:49.019915 28752 tablet.cc:1722] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:49.020190 28752 tablet.cc:1620] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:58:49.020459 28320 tablet.cc:1722] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:49.020792 28320 tablet.cc:1620] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e: MemRowSet was empty: no flush needed.
I20240502 07:58:49.023847 22438 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=3bcedb75536c4dc9b765c1f785166da1] alter complete (version 2)
I20240502 07:58:49.024435 27339 raft_consensus.cc:1050] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537: attempting to promote NON_VOTER 7db4178eed174225bfb24f5f9445ce88 to VOTER
I20240502 07:58:49.024874 27339 consensus_queue.cc:237] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } }
I20240502 07:58:49.026487 25265 raft_consensus.cc:1261] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:49.026731 25134 raft_consensus.cc:1261] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:49.027058 28822 consensus_queue.cc:1035] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:49.027316 22845 raft_consensus.cc:1261] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:49.027523 27338 consensus_queue.cc:1035] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:49.027894 27338 consensus_queue.cc:1035] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:49.029162 25266 raft_consensus.cc:1203] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 07:58:49.031126 28824 raft_consensus.cc:2932] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:49.031198 25265 raft_consensus.cc:2932] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:49.033411 22845 raft_consensus.cc:2932] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:49.033427 25135 raft_consensus.cc:2932] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:49.034029 22438 catalog_manager.cc:5484] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:49.039232 24871 consensus_queue.cc:237] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } }
I20240502 07:58:49.040166 22845 raft_consensus.cc:1261] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:49.040220 25135 raft_consensus.cc:1261] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:49.040524 28823 consensus_queue.cc:1035] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:49.040735 28824 consensus_queue.cc:1035] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:49.042367 28823 raft_consensus.cc:2932] T 3bb108b6ab55461fb9fabf431d953ef2 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:49.042935 22845 raft_consensus.cc:2932] T 3bb108b6ab55461fb9fabf431d953ef2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:49.043502 25134 raft_consensus.cc:2932] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:49.044860 22428 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3bb108b6ab55461fb9fabf431d953ef2 with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:49.045899 22444 catalog_manager.cc:5484] T 3bb108b6ab55461fb9fabf431d953ef2 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index 3 to 4, VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) evicted. New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:49.048347 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet 3bb108b6ab55461fb9fabf431d953ef2 with delete_type TABLET_DATA_TOMBSTONED (TS 7eced220cf1642f3873fc385b6191e69 not found in new config with opid_index 4) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:49.048597 28510 tablet_replica.cc:330] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:49.048769 28510 raft_consensus.cc:2229] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:49.048884 28510 raft_consensus.cc:2258] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:49.050208 28510 ts_tablet_manager.cc:1875] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:49.051903 28510 ts_tablet_manager.cc:1888] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.3
I20240502 07:58:49.051962 28510 log.cc:1196] T 3bb108b6ab55461fb9fabf431d953ef2 P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/3bb108b6ab55461fb9fabf431d953ef2
I20240502 07:58:49.052435 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet 3bb108b6ab55461fb9fabf431d953ef2 (table rebalancer_extra_table_10 [id=a3c8266f3e52403ca929be65617dd5f3]) successfully deleted
I20240502 07:58:49.069334 28648 ts_tablet_manager.cc:902] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:49.069756 28648 tablet_copy_client.cc:308] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33387
I20240502 07:58:49.073436 22737 tablet_copy_service.cc:140] P 725969d490a04fdfb3ba890c3f5096e3: Received BeginTabletCopySession request for tablet 023958ce03b84b2da063c13e4284e04d from peer 6d02017318a2461796d431962fe6a537 ({username='slave'} at 127.16.85.133:53877)
I20240502 07:58:49.073540 22737 tablet_copy_service.cc:161] P 725969d490a04fdfb3ba890c3f5096e3: Beginning new tablet copy session on tablet 023958ce03b84b2da063c13e4284e04d from peer 6d02017318a2461796d431962fe6a537 at {username='slave'} at 127.16.85.133:53877: session id = 6d02017318a2461796d431962fe6a537-023958ce03b84b2da063c13e4284e04d
I20240502 07:58:49.074385 22737 tablet_copy_source_session.cc:205] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:49.075292 28648 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 023958ce03b84b2da063c13e4284e04d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.077524 28648 tablet_copy_client.cc:783] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:49.077682 28648 tablet_copy_client.cc:647] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:49.078790 28648 tablet_copy_client.cc:523] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:49.079748 28648 tablet_bootstrap.cc:492] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:49.088378 27808 consensus_queue.cc:1035] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:49.091835 28205 consensus_queue.cc:1035] T e032524f34e04ffea0c001e403e840c9 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:49.106452 28648 tablet_bootstrap.cc:492] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:49.106992 28648 tablet_bootstrap.cc:492] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537: Bootstrap complete.
I20240502 07:58:49.107170 28648 ts_tablet_manager.cc:1367] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.027s user 0.025s sys 0.000s
I20240502 07:58:49.107640 28648 raft_consensus.cc:348] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:49.107784 28648 raft_consensus.cc:729] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: LEARNER
I20240502 07:58:49.107888 28648 consensus_queue.cc:260] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 07:58:49.108156 28648 ts_tablet_manager.cc:1398] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:49.108619 22737 tablet_copy_service.cc:342] P 725969d490a04fdfb3ba890c3f5096e3: Request end of tablet copy session 6d02017318a2461796d431962fe6a537-023958ce03b84b2da063c13e4284e04d received from {username='slave'} at 127.16.85.133:53877
I20240502 07:58:49.108726 22737 tablet_copy_service.cc:433] P 725969d490a04fdfb3ba890c3f5096e3: ending tablet copy session 6d02017318a2461796d431962fe6a537-023958ce03b84b2da063c13e4284e04d on tablet 023958ce03b84b2da063c13e4284e04d with peer 6d02017318a2461796d431962fe6a537
I20240502 07:58:49.127547 22438 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:49.129148 22716 consensus_queue.cc:237] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
W20240502 07:58:49.130683 22652 consensus_peers.cc:485] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 68f96ca3354645fd91f666109eeaf847. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:49.131469 22845 raft_consensus.cc:1261] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:49.132108 28450 consensus_queue.cc:1035] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:49.132956 28320 tablet.cc:1722] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:49.133128 22586 raft_consensus.cc:1261] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:49.133296 28751 tablet.cc:1722] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:49.134804 28752 tablet.cc:1722] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:49.135680 28450 consensus_queue.cc:1035] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:49.136741 28450 raft_consensus.cc:2932] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:49.138442 22845 raft_consensus.cc:2932] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:49.139325 22438 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=3bcedb75536c4dc9b765c1f785166da1] alter complete (version 3)
I20240502 07:58:49.139343 22586 raft_consensus.cc:2932] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:49.141208 22444 catalog_manager.cc:5484] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 4, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) added. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } } }
I20240502 07:58:49.144027 28918 ts_tablet_manager.cc:902] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:49.144513 28918 tablet_copy_client.cc:308] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:49.144986 22999 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet f1fa312fe8e44d23970c5e3ba98c2c58 from peer 4654231dfe994c1a9d2e37d9c3d75699 ({username='slave'} at 127.16.85.135:41731)
I20240502 07:58:49.145099 22999 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet f1fa312fe8e44d23970c5e3ba98c2c58 from peer 4654231dfe994c1a9d2e37d9c3d75699 at {username='slave'} at 127.16.85.135:41731: session id = 4654231dfe994c1a9d2e37d9c3d75699-f1fa312fe8e44d23970c5e3ba98c2c58
I20240502 07:58:49.145974 22999 tablet_copy_source_session.cc:205] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:49.146791 28918 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f1fa312fe8e44d23970c5e3ba98c2c58. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.149446 28918 tablet_copy_client.cc:783] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:49.149601 28918 tablet_copy_client.cc:647] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:49.150612 28918 tablet_copy_client.cc:523] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:49.151373 22717 consensus_queue.cc:237] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 2.7, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:49.151687 28918 tablet_bootstrap.cc:492] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:49.152997 24871 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 2 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 2 index: 7. Preceding OpId from leader: term: 2 index: 8. (index mismatch)
W20240502 07:58:49.153095 22652 consensus_peers.cc:485] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Couldn't send request to peer 7eced220cf1642f3873fc385b6191e69. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9d2283e5d28347519b38e876d840608e. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:49.153318 28823 consensus_queue.cc:1035] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:49.153432 28470 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:49.153630 22845 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 2 index: 7. Preceding OpId from leader: term: 2 index: 8. (index mismatch)
I20240502 07:58:49.154058 28470 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:49.155594 26995 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 2 LEADER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:49.159652 22435 catalog_manager.cc:5484] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 7 to 8, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New cstate: current_term: 2 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:49.160840 28823 consensus_queue.cc:1035] T 6c781176e9324a64968f9dcb526c532b P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:49.161381 24871 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 2 FOLLOWER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:49.167614 22845 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:49.178637 28918 tablet_bootstrap.cc:492] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:49.178903 28918 tablet_bootstrap.cc:492] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap complete.
I20240502 07:58:49.179059 28918 ts_tablet_manager.cc:1367] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.027s user 0.025s sys 0.000s
I20240502 07:58:49.179415 28918 raft_consensus.cc:348] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:49.179535 28918 raft_consensus.cc:729] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: LEARNER
I20240502 07:58:49.179626 28918 consensus_queue.cc:260] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:49.179879 28918 ts_tablet_manager.cc:1398] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:49.180272 22999 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-f1fa312fe8e44d23970c5e3ba98c2c58 received from {username='slave'} at 127.16.85.135:41731
I20240502 07:58:49.180344 22999 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-f1fa312fe8e44d23970c5e3ba98c2c58 on tablet f1fa312fe8e44d23970c5e3ba98c2c58 with peer 4654231dfe994c1a9d2e37d9c3d75699
I20240502 07:58:49.225646 25003 raft_consensus.cc:1203] T da39029564d042f6acfc23f99fb63dfd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 07:58:49.227671 28989 ts_tablet_manager.cc:902] T bd1c2abfe2a3416bb10d6c14d3fed46a P 74f9aa91c2c448f980b512fe2e8d672e: Initiating tablet copy from peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:49.228842 28989 tablet_copy_client.cc:308] T bd1c2abfe2a3416bb10d6c14d3fed46a P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.131:44055
I20240502 07:58:49.229243 22868 tablet_copy_service.cc:140] P 7db4178eed174225bfb24f5f9445ce88: Received BeginTabletCopySession request for tablet bd1c2abfe2a3416bb10d6c14d3fed46a from peer 74f9aa91c2c448f980b512fe2e8d672e ({username='slave'} at 127.16.85.134:53609)
I20240502 07:58:49.229336 22868 tablet_copy_service.cc:161] P 7db4178eed174225bfb24f5f9445ce88: Beginning new tablet copy session on tablet bd1c2abfe2a3416bb10d6c14d3fed46a from peer 74f9aa91c2c448f980b512fe2e8d672e at {username='slave'} at 127.16.85.134:53609: session id = 74f9aa91c2c448f980b512fe2e8d672e-bd1c2abfe2a3416bb10d6c14d3fed46a
I20240502 07:58:49.230023 22868 tablet_copy_source_session.cc:205] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:49.230799 28989 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd1c2abfe2a3416bb10d6c14d3fed46a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.233068 28989 tablet_copy_client.cc:783] T bd1c2abfe2a3416bb10d6c14d3fed46a P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:49.233211 28989 tablet_copy_client.cc:647] T bd1c2abfe2a3416bb10d6c14d3fed46a P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:49.234289 28989 tablet_copy_client.cc:523] T bd1c2abfe2a3416bb10d6c14d3fed46a P 74f9aa91c2c448f980b512fe2e8d672e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:49.235205 28989 tablet_bootstrap.cc:492] T bd1c2abfe2a3416bb10d6c14d3fed46a P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:49.242115 22435 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:49.242255 22435 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:49.244150 22435 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet cf2982b56cb24afca4ada89a8cbd6645
I20240502 07:58:49.244632 25115 tablet_service.cc:1514] Processing DeleteTablet for tablet cf2982b56cb24afca4ada89a8cbd6645 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:49 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:49.244925 29005 tablet_replica.cc:330] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:49.244966 24983 tablet_service.cc:1514] Processing DeleteTablet for tablet cf2982b56cb24afca4ada89a8cbd6645 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:49 UTC) from {username='slave'} at 127.0.0.1:42878
I20240502 07:58:49.244994 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet cf2982b56cb24afca4ada89a8cbd6645 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:49 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:49.245162 28353 tablet_replica.cc:330] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:49.245277 28353 raft_consensus.cc:2229] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:49.245298 29005 raft_consensus.cc:2229] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:49.245383 28353 raft_consensus.cc:2258] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:49.245491 29005 raft_consensus.cc:2258] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:49.245922 28353 ts_tablet_manager.cc:1875] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:49.246016 29005 ts_tablet_manager.cc:1875] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:49.248051 29005 ts_tablet_manager.cc:1888] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:49.248116 29005 log.cc:1196] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/cf2982b56cb24afca4ada89a8cbd6645
I20240502 07:58:49.248131 28353 ts_tablet_manager.cc:1888] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:49.248200 28353 log.cc:1196] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/cf2982b56cb24afca4ada89a8cbd6645
I20240502 07:58:49.248366 29005 ts_tablet_manager.cc:1909] T cf2982b56cb24afca4ada89a8cbd6645 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:49.248467 28353 ts_tablet_manager.cc:1909] T cf2982b56cb24afca4ada89a8cbd6645 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:49.248960 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet cf2982b56cb24afca4ada89a8cbd6645 (table rebalancer_dynamic_table_new_name [id=3bcedb75536c4dc9b765c1f785166da1]) successfully deleted
I20240502 07:58:49.249329 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet cf2982b56cb24afca4ada89a8cbd6645 (table rebalancer_dynamic_table_new_name [id=3bcedb75536c4dc9b765c1f785166da1]) successfully deleted
I20240502 07:58:49.249356 29006 tablet_replica.cc:330] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e: stopping tablet replica
I20240502 07:58:49.249529 29006 raft_consensus.cc:2229] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:49.249655 29006 raft_consensus.cc:2258] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:49.250114 29006 ts_tablet_manager.cc:1875] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:49.252163 29006 ts_tablet_manager.cc:1888] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:49.252228 29006 log.cc:1196] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-5/wal/wals/cf2982b56cb24afca4ada89a8cbd6645
I20240502 07:58:49.252475 29006 ts_tablet_manager.cc:1909] T cf2982b56cb24afca4ada89a8cbd6645 P 74f9aa91c2c448f980b512fe2e8d672e: Deleting consensus metadata
I20240502 07:58:49.253080 22430 catalog_manager.cc:4830] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): tablet cf2982b56cb24afca4ada89a8cbd6645 (table rebalancer_dynamic_table_new_name [id=3bcedb75536c4dc9b765c1f785166da1]) successfully deleted
I20240502 07:58:49.257777 22978 raft_consensus.cc:1203] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 07:58:49.258615 28989 tablet_bootstrap.cc:492] T bd1c2abfe2a3416bb10d6c14d3fed46a P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:49.258870 28989 tablet_bootstrap.cc:492] T bd1c2abfe2a3416bb10d6c14d3fed46a P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap complete.
I20240502 07:58:49.259019 28989 ts_tablet_manager.cc:1367] T bd1c2abfe2a3416bb10d6c14d3fed46a P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.024s user 0.021s sys 0.000s
I20240502 07:58:49.259434 28989 raft_consensus.cc:348] T bd1c2abfe2a3416bb10d6c14d3fed46a P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:49.259546 28989 raft_consensus.cc:729] T bd1c2abfe2a3416bb10d6c14d3fed46a P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: LEARNER
I20240502 07:58:49.259646 28989 consensus_queue.cc:260] T bd1c2abfe2a3416bb10d6c14d3fed46a P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: NON_VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: true } }
I20240502 07:58:49.259941 28989 ts_tablet_manager.cc:1398] T bd1c2abfe2a3416bb10d6c14d3fed46a P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.003s sys 0.000s
I20240502 07:58:49.260493 22868 tablet_copy_service.cc:342] P 7db4178eed174225bfb24f5f9445ce88: Request end of tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-bd1c2abfe2a3416bb10d6c14d3fed46a received from {username='slave'} at 127.16.85.134:53609
I20240502 07:58:49.260583 22868 tablet_copy_service.cc:433] P 7db4178eed174225bfb24f5f9445ce88: ending tablet copy session 74f9aa91c2c448f980b512fe2e8d672e-bd1c2abfe2a3416bb10d6c14d3fed46a on tablet bd1c2abfe2a3416bb10d6c14d3fed46a with peer 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:49.306461 25265 raft_consensus.cc:1261] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69 [term 2 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 2 index: 4. (index mismatch)
I20240502 07:58:49.306852 27023 consensus_queue.cc:1035] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:49.317134 29007 ts_tablet_manager.cc:902] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69: Initiating tablet copy from peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:49.317652 29007 tablet_copy_client.cc:308] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:37451
I20240502 07:58:49.318090 22606 tablet_copy_service.cc:140] P 8bea0fe8d8044a2685f92f7a10c9cf85: Received BeginTabletCopySession request for tablet 9e5fe86b44744df9b2e4d430fdec716f from peer 7eced220cf1642f3873fc385b6191e69 ({username='slave'} at 127.16.85.136:57807)
I20240502 07:58:49.318184 22606 tablet_copy_service.cc:161] P 8bea0fe8d8044a2685f92f7a10c9cf85: Beginning new tablet copy session on tablet 9e5fe86b44744df9b2e4d430fdec716f from peer 7eced220cf1642f3873fc385b6191e69 at {username='slave'} at 127.16.85.136:57807: session id = 7eced220cf1642f3873fc385b6191e69-9e5fe86b44744df9b2e4d430fdec716f
I20240502 07:58:49.318922 22606 tablet_copy_source_session.cc:205] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:49.319681 29007 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9e5fe86b44744df9b2e4d430fdec716f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.321452 29007 tablet_copy_client.cc:783] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:49.321599 29007 tablet_copy_client.cc:647] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:49.322496 29007 tablet_copy_client.cc:523] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:49.323597 29007 tablet_bootstrap.cc:492] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:49.341081 29007 tablet_bootstrap.cc:492] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:49.341279 29007 tablet_bootstrap.cc:492] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69: Bootstrap complete.
I20240502 07:58:49.341404 29007 ts_tablet_manager.cc:1367] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.018s user 0.018s sys 0.000s
I20240502 07:58:49.341744 29007 raft_consensus.cc:348] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:49.341862 29007 raft_consensus.cc:729] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: LEARNER
I20240502 07:58:49.341964 29007 consensus_queue.cc:260] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:49.342212 29007 ts_tablet_manager.cc:1398] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:49.342610 22606 tablet_copy_service.cc:342] P 8bea0fe8d8044a2685f92f7a10c9cf85: Request end of tablet copy session 7eced220cf1642f3873fc385b6191e69-9e5fe86b44744df9b2e4d430fdec716f received from {username='slave'} at 127.16.85.136:57807
I20240502 07:58:49.342689 22606 tablet_copy_service.cc:433] P 8bea0fe8d8044a2685f92f7a10c9cf85: ending tablet copy session 7eced220cf1642f3873fc385b6191e69-9e5fe86b44744df9b2e4d430fdec716f on tablet 9e5fe86b44744df9b2e4d430fdec716f with peer 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:49.346139 22438 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:49.350366 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 73dd5ed5e23b44839c956ffa01d69345 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=f01f3d0fc0ba4997b40287d9e08e8fc3]), partition=
I20240502 07:58:49.350528 24983 tablet_service.cc:1467] Processing CreateTablet for tablet 73dd5ed5e23b44839c956ffa01d69345 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=f01f3d0fc0ba4997b40287d9e08e8fc3]), partition=
I20240502 07:58:49.350585 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 73dd5ed5e23b44839c956ffa01d69345. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.350669 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 73dd5ed5e23b44839c956ffa01d69345 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=f01f3d0fc0ba4997b40287d9e08e8fc3]), partition=
I20240502 07:58:49.350764 24983 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 73dd5ed5e23b44839c956ffa01d69345. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.350910 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 73dd5ed5e23b44839c956ffa01d69345. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.352947 29013 tablet_bootstrap.cc:492] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:49.353268 29014 tablet_bootstrap.cc:492] T 73dd5ed5e23b44839c956ffa01d69345 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:49.353824 29013 tablet_bootstrap.cc:654] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:49.354326 29014 tablet_bootstrap.cc:654] T 73dd5ed5e23b44839c956ffa01d69345 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:49.354694 29013 tablet_bootstrap.cc:492] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:49.354764 29013 ts_tablet_manager.cc:1367] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:49.355160 29013 raft_consensus.cc:348] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.355301 29014 tablet_bootstrap.cc:492] T 73dd5ed5e23b44839c956ffa01d69345 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:49.355314 29013 raft_consensus.cc:374] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:49.355365 29013 raft_consensus.cc:729] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:49.355389 29014 ts_tablet_manager.cc:1367] T 73dd5ed5e23b44839c956ffa01d69345 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:49.355455 29013 consensus_queue.cc:260] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.355680 29013 ts_tablet_manager.cc:1398] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:49.355767 29014 raft_consensus.cc:348] T 73dd5ed5e23b44839c956ffa01d69345 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.355870 29014 raft_consensus.cc:374] T 73dd5ed5e23b44839c956ffa01d69345 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:49.355895 29014 raft_consensus.cc:729] T 73dd5ed5e23b44839c956ffa01d69345 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:49.355991 29014 consensus_queue.cc:260] T 73dd5ed5e23b44839c956ffa01d69345 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.356240 29014 ts_tablet_manager.cc:1398] T 73dd5ed5e23b44839c956ffa01d69345 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:49.356477 29015 tablet_bootstrap.cc:492] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:49.357373 29015 tablet_bootstrap.cc:654] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:49.358667 29015 tablet_bootstrap.cc:492] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:49.358800 29015 ts_tablet_manager.cc:1367] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:49.359228 29015 raft_consensus.cc:348] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.359331 29015 raft_consensus.cc:374] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:49.359401 29015 raft_consensus.cc:729] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:49.359537 29015 consensus_queue.cc:260] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.359963 29015 ts_tablet_manager.cc:1398] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:49.365830 27002 consensus_queue.cc:1035] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:49.368460 27808 consensus_queue.cc:1035] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:49.376302 22717 raft_consensus.cc:1203] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 07:58:49.402040 24871 raft_consensus.cc:1203] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 07:58:49.445541 25134 raft_consensus.cc:1261] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699 [term 2 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 2 index: 4. (index mismatch)
I20240502 07:58:49.446069 27023 consensus_queue.cc:1035] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:49.448748 22845 raft_consensus.cc:1261] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 2 index: 4. (index mismatch)
W20240502 07:58:49.450639 27023 consensus_queue.cc:1159] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "4654231dfe994c1a9d2e37d9c3d75699" responder_term: 2 status { last_received { term: 2 index: 4 } last_committed_idx: 4 last_received_current_leader { term: 2 index: 4 } }
I20240502 07:58:49.450779 22584 consensus_queue.cc:237] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 2.4, Last appended by leader: 3, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: false } }
I20240502 07:58:49.452059 25265 raft_consensus.cc:1261] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69 [term 2 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 2 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
I20240502 07:58:49.452481 28636 consensus_queue.cc:1035] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:49.452615 25134 raft_consensus.cc:1261] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699 [term 2 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 2 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
I20240502 07:58:49.452940 28636 consensus_queue.cc:1035] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:49.454319 27023 raft_consensus.cc:2932] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 2 LEADER]: Committing config change with OpId 2.5: config changed from index 3 to 5, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: false } } }
I20240502 07:58:49.455345 25134 raft_consensus.cc:2932] T 66ac69e448574c7ea5101014e0dd425d P 4654231dfe994c1a9d2e37d9c3d75699 [term 2 FOLLOWER]: Committing config change with OpId 2.5: config changed from index 3 to 5, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: false } } }
I20240502 07:58:49.455682 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 66ac69e448574c7ea5101014e0dd425d with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:49.455742 25265 raft_consensus.cc:2932] T 66ac69e448574c7ea5101014e0dd425d P 7eced220cf1642f3873fc385b6191e69 [term 2 FOLLOWER]: Committing config change with OpId 2.5: config changed from index 3 to 5, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: false } } }
I20240502 07:58:49.457578 22435 catalog_manager.cc:5484] T 66ac69e448574c7ea5101014e0dd425d P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 3 to 5, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New cstate: current_term: 2 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:49.460911 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet 66ac69e448574c7ea5101014e0dd425d with delete_type TABLET_DATA_TOMBSTONED (TS 7db4178eed174225bfb24f5f9445ce88 not found in new config with opid_index 5) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:49.461061 27136 raft_consensus.cc:1050] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85: attempting to promote NON_VOTER 7eced220cf1642f3873fc385b6191e69 to VOTER
I20240502 07:58:49.461370 29034 tablet_replica.cc:330] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:49.461400 27136 consensus_queue.cc:237] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:49.461536 29034 raft_consensus.cc:2229] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:49.461632 29034 raft_consensus.cc:2258] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:49.462044 29034 ts_tablet_manager.cc:1875] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:49.463624 25134 raft_consensus.cc:1261] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:49.463712 22845 raft_consensus.cc:1261] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:49.464083 28636 consensus_queue.cc:1035] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:49.464227 27023 consensus_queue.cc:1035] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:49.464524 29034 ts_tablet_manager.cc:1888] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.3
I20240502 07:58:49.464601 29034 log.cc:1196] T 66ac69e448574c7ea5101014e0dd425d P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/66ac69e448574c7ea5101014e0dd425d
I20240502 07:58:49.465232 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 66ac69e448574c7ea5101014e0dd425d (table rebalancer_extra_table_6 [id=4aefdd54f1e145688b445ce3d02fb21e]) successfully deleted
I20240502 07:58:49.467154 25265 raft_consensus.cc:1261] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:49.467550 28636 consensus_queue.cc:1035] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:49.467856 27023 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:49.468456 25265 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:49.469244 22845 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:49.470443 25134 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:49.471175 22435 catalog_manager.cc:5484] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:49.485100 22438 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_18"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:49.487478 29007 ts_tablet_manager.cc:902] T a8dcc36740be4ca3ac1d43591f7549bd P 7eced220cf1642f3873fc385b6191e69: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:49.487907 29007 tablet_copy_client.cc:308] T a8dcc36740be4ca3ac1d43591f7549bd P 7eced220cf1642f3873fc385b6191e69: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:49.488291 22999 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet a8dcc36740be4ca3ac1d43591f7549bd from peer 7eced220cf1642f3873fc385b6191e69 ({username='slave'} at 127.16.85.136:40185)
I20240502 07:58:49.488373 22999 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet a8dcc36740be4ca3ac1d43591f7549bd from peer 7eced220cf1642f3873fc385b6191e69 at {username='slave'} at 127.16.85.136:40185: session id = 7eced220cf1642f3873fc385b6191e69-a8dcc36740be4ca3ac1d43591f7549bd
I20240502 07:58:49.489312 22999 tablet_copy_source_session.cc:205] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:49.490051 29007 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a8dcc36740be4ca3ac1d43591f7549bd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.492153 29007 tablet_copy_client.cc:783] T a8dcc36740be4ca3ac1d43591f7549bd P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:49.492283 29007 tablet_copy_client.cc:647] T a8dcc36740be4ca3ac1d43591f7549bd P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:49.492643 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 4679a4f9194a4dbcb85c28e31f6ac894 (DEFAULT_TABLE table=rebalancer_extra_table_18 [id=6fc6a271776a4ac4ab42baf165958986]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:49.492926 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4679a4f9194a4dbcb85c28e31f6ac894. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.493129 22697 tablet_service.cc:1467] Processing CreateTablet for tablet cf0d3af4df9b410baaa088c744eeb2a6 (DEFAULT_TABLE table=rebalancer_extra_table_18 [id=6fc6a271776a4ac4ab42baf165958986]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:49.493371 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf0d3af4df9b410baaa088c744eeb2a6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.495023 29063 tablet_bootstrap.cc:492] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:49.495396 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 4679a4f9194a4dbcb85c28e31f6ac894 (DEFAULT_TABLE table=rebalancer_extra_table_18 [id=6fc6a271776a4ac4ab42baf165958986]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:49.495416 22827 tablet_service.cc:1467] Processing CreateTablet for tablet c41dd9727c3f44cbae17bad891e55636 (DEFAULT_TABLE table=rebalancer_extra_table_18 [id=6fc6a271776a4ac4ab42baf165958986]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:49.495620 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4679a4f9194a4dbcb85c28e31f6ac894. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.495644 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c41dd9727c3f44cbae17bad891e55636. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.495795 29063 tablet_bootstrap.cc:654] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:49.495755 24850 tablet_service.cc:1467] Processing CreateTablet for tablet c41dd9727c3f44cbae17bad891e55636 (DEFAULT_TABLE table=rebalancer_extra_table_18 [id=6fc6a271776a4ac4ab42baf165958986]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:49.495950 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c41dd9727c3f44cbae17bad891e55636. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.496325 29064 tablet_bootstrap.cc:492] T cf0d3af4df9b410baaa088c744eeb2a6 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:49.497294 25246 tablet_service.cc:1467] Processing CreateTablet for tablet c41dd9727c3f44cbae17bad891e55636 (DEFAULT_TABLE table=rebalancer_extra_table_18 [id=6fc6a271776a4ac4ab42baf165958986]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:49.497522 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c41dd9727c3f44cbae17bad891e55636. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.497531 24984 tablet_service.cc:1467] Processing CreateTablet for tablet cf0d3af4df9b410baaa088c744eeb2a6 (DEFAULT_TABLE table=rebalancer_extra_table_18 [id=6fc6a271776a4ac4ab42baf165958986]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:49.497756 24984 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf0d3af4df9b410baaa088c744eeb2a6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.498056 24983 tablet_service.cc:1467] Processing CreateTablet for tablet 4679a4f9194a4dbcb85c28e31f6ac894 (DEFAULT_TABLE table=rebalancer_extra_table_18 [id=6fc6a271776a4ac4ab42baf165958986]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:49.498242 24983 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4679a4f9194a4dbcb85c28e31f6ac894. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.499122 29013 tablet_bootstrap.cc:492] T c41dd9727c3f44cbae17bad891e55636 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:49.499284 25235 tablet_service.cc:1467] Processing CreateTablet for tablet cf0d3af4df9b410baaa088c744eeb2a6 (DEFAULT_TABLE table=rebalancer_extra_table_18 [id=6fc6a271776a4ac4ab42baf165958986]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:49.499461 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf0d3af4df9b410baaa088c744eeb2a6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.499711 29014 tablet_bootstrap.cc:492] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:49.500000 29015 tablet_bootstrap.cc:492] T 4679a4f9194a4dbcb85c28e31f6ac894 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:49.500247 29013 tablet_bootstrap.cc:654] T c41dd9727c3f44cbae17bad891e55636 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:49.500525 29014 tablet_bootstrap.cc:654] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:49.500895 29015 tablet_bootstrap.cc:654] T 4679a4f9194a4dbcb85c28e31f6ac894 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:49.502696 29007 tablet_copy_client.cc:523] T a8dcc36740be4ca3ac1d43591f7549bd P 7eced220cf1642f3873fc385b6191e69: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:49.502842 29063 tablet_bootstrap.cc:492] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:49.502907 29063 ts_tablet_manager.cc:1367] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.000s
I20240502 07:58:49.503490 29069 tablet_bootstrap.cc:492] T c41dd9727c3f44cbae17bad891e55636 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:49.504027 29007 tablet_bootstrap.cc:492] T a8dcc36740be4ca3ac1d43591f7549bd P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:49.504489 29063 raft_consensus.cc:348] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.504824 29063 raft_consensus.cc:374] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:49.504926 29069 tablet_bootstrap.cc:654] T c41dd9727c3f44cbae17bad891e55636 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:49.504926 29063 raft_consensus.cc:729] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:49.505062 29063 consensus_queue.cc:260] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.504635 29014 tablet_bootstrap.cc:492] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:49.505545 29014 ts_tablet_manager.cc:1367] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:58:49.505928 29069 tablet_bootstrap.cc:492] T c41dd9727c3f44cbae17bad891e55636 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:49.505998 29069 ts_tablet_manager.cc:1367] T c41dd9727c3f44cbae17bad891e55636 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:49.505961 29014 raft_consensus.cc:348] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:49.506053 29014 raft_consensus.cc:374] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:49.506083 29014 raft_consensus.cc:729] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:49.506227 29014 consensus_queue.cc:260] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:49.506410 29069 raft_consensus.cc:348] T c41dd9727c3f44cbae17bad891e55636 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:49.506500 29069 raft_consensus.cc:374] T c41dd9727c3f44cbae17bad891e55636 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:49.506525 29069 raft_consensus.cc:729] T c41dd9727c3f44cbae17bad891e55636 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:49.506527 29014 ts_tablet_manager.cc:1398] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:49.506671 29014 tablet_bootstrap.cc:492] T 4679a4f9194a4dbcb85c28e31f6ac894 P 74f9aa91c2c448f980b512fe2e8d672e: Bootstrap starting.
I20240502 07:58:49.506640 29069 consensus_queue.cc:260] T c41dd9727c3f44cbae17bad891e55636 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:49.506907 29069 ts_tablet_manager.cc:1398] T c41dd9727c3f44cbae17bad891e55636 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:49.507400 29063 ts_tablet_manager.cc:1398] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:49.507432 29014 tablet_bootstrap.cc:654] T 4679a4f9194a4dbcb85c28e31f6ac894 P 74f9aa91c2c448f980b512fe2e8d672e: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:49.508617 29064 tablet_bootstrap.cc:654] T cf0d3af4df9b410baaa088c744eeb2a6 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:49.509694 29064 tablet_bootstrap.cc:492] T cf0d3af4df9b410baaa088c744eeb2a6 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:49.509831 29015 tablet_bootstrap.cc:492] T 4679a4f9194a4dbcb85c28e31f6ac894 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:49.509843 29064 ts_tablet_manager.cc:1367] T cf0d3af4df9b410baaa088c744eeb2a6 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.014s user 0.001s sys 0.001s
I20240502 07:58:49.509913 29015 ts_tablet_manager.cc:1367] T 4679a4f9194a4dbcb85c28e31f6ac894 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.010s user 0.001s sys 0.000s
I20240502 07:58:49.510354 29013 tablet_bootstrap.cc:492] T c41dd9727c3f44cbae17bad891e55636 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:49.510313 29015 raft_consensus.cc:348] T 4679a4f9194a4dbcb85c28e31f6ac894 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.510399 29015 raft_consensus.cc:374] T 4679a4f9194a4dbcb85c28e31f6ac894 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:49.510428 29015 raft_consensus.cc:729] T 4679a4f9194a4dbcb85c28e31f6ac894 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:49.510430 29013 ts_tablet_manager.cc:1367] T c41dd9727c3f44cbae17bad891e55636 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.000s
I20240502 07:58:49.510449 29064 raft_consensus.cc:348] T cf0d3af4df9b410baaa088c744eeb2a6 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:49.510571 29064 raft_consensus.cc:374] T cf0d3af4df9b410baaa088c744eeb2a6 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:49.510545 29015 consensus_queue.cc:260] T 4679a4f9194a4dbcb85c28e31f6ac894 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.510599 29064 raft_consensus.cc:729] T cf0d3af4df9b410baaa088c744eeb2a6 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:49.510732 29064 consensus_queue.cc:260] T cf0d3af4df9b410baaa088c744eeb2a6 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:49.510811 29015 ts_tablet_manager.cc:1398] T 4679a4f9194a4dbcb85c28e31f6ac894 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:49.510841 29013 raft_consensus.cc:348] T c41dd9727c3f44cbae17bad891e55636 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:49.510959 29013 raft_consensus.cc:374] T c41dd9727c3f44cbae17bad891e55636 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:49.510996 29013 raft_consensus.cc:729] T c41dd9727c3f44cbae17bad891e55636 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:49.511045 29064 ts_tablet_manager.cc:1398] T cf0d3af4df9b410baaa088c744eeb2a6 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:49.511114 29015 tablet_bootstrap.cc:492] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:49.511101 29013 consensus_queue.cc:260] T c41dd9727c3f44cbae17bad891e55636 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:49.511345 29013 ts_tablet_manager.cc:1398] T c41dd9727c3f44cbae17bad891e55636 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:49.511512 29013 tablet_bootstrap.cc:492] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:49.512395 29013 tablet_bootstrap.cc:654] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:49.512864 29015 tablet_bootstrap.cc:654] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:49.513739 29014 tablet_bootstrap.cc:492] T 4679a4f9194a4dbcb85c28e31f6ac894 P 74f9aa91c2c448f980b512fe2e8d672e: No bootstrap required, opened a new log
I20240502 07:58:49.513816 29014 ts_tablet_manager.cc:1367] T 4679a4f9194a4dbcb85c28e31f6ac894 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 07:58:49.514245 29014 raft_consensus.cc:348] T 4679a4f9194a4dbcb85c28e31f6ac894 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.514329 29014 raft_consensus.cc:374] T 4679a4f9194a4dbcb85c28e31f6ac894 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:49.514357 29014 raft_consensus.cc:729] T 4679a4f9194a4dbcb85c28e31f6ac894 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Initialized, Role: FOLLOWER
I20240502 07:58:49.514461 29014 consensus_queue.cc:260] T 4679a4f9194a4dbcb85c28e31f6ac894 P 74f9aa91c2c448f980b512fe2e8d672e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.514757 29014 ts_tablet_manager.cc:1398] T 4679a4f9194a4dbcb85c28e31f6ac894 P 74f9aa91c2c448f980b512fe2e8d672e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:49.517270 29015 tablet_bootstrap.cc:492] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:49.517354 29015 ts_tablet_manager.cc:1367] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:58:49.517729 29015 raft_consensus.cc:348] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:49.517848 29015 raft_consensus.cc:374] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:49.517879 29015 raft_consensus.cc:729] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:49.517992 29015 consensus_queue.cc:260] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:49.518253 29015 ts_tablet_manager.cc:1398] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:49.519757 29013 tablet_bootstrap.cc:492] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:49.519891 29013 ts_tablet_manager.cc:1367] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.000s
I20240502 07:58:49.520422 29013 raft_consensus.cc:348] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:49.520555 29013 raft_consensus.cc:374] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:49.520620 29013 raft_consensus.cc:729] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:49.520792 29013 consensus_queue.cc:260] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:49.521785 29013 ts_tablet_manager.cc:1398] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:49.537497 29007 tablet_bootstrap.cc:492] T a8dcc36740be4ca3ac1d43591f7549bd P 7eced220cf1642f3873fc385b6191e69: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:49.537750 29007 tablet_bootstrap.cc:492] T a8dcc36740be4ca3ac1d43591f7549bd P 7eced220cf1642f3873fc385b6191e69: Bootstrap complete.
I20240502 07:58:49.537981 29007 ts_tablet_manager.cc:1367] T a8dcc36740be4ca3ac1d43591f7549bd P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.034s user 0.026s sys 0.000s
I20240502 07:58:49.538429 29007 raft_consensus.cc:348] T a8dcc36740be4ca3ac1d43591f7549bd P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:49.538645 29007 raft_consensus.cc:729] T a8dcc36740be4ca3ac1d43591f7549bd P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: LEARNER
I20240502 07:58:49.538798 29007 consensus_queue.cc:260] T a8dcc36740be4ca3ac1d43591f7549bd P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:49.539222 29007 ts_tablet_manager.cc:1398] T a8dcc36740be4ca3ac1d43591f7549bd P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:49.539851 22999 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 7eced220cf1642f3873fc385b6191e69-a8dcc36740be4ca3ac1d43591f7549bd received from {username='slave'} at 127.16.85.136:40185
I20240502 07:58:49.539928 22999 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 7eced220cf1642f3873fc385b6191e69-a8dcc36740be4ca3ac1d43591f7549bd on tablet a8dcc36740be4ca3ac1d43591f7549bd with peer 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:49.553120 26696 raft_consensus.cc:482] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:49.553210 26696 raft_consensus.cc:504] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:49.553651 26696 leader_election.cc:288] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:49.553941 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0d3af4df9b410baaa088c744eeb2a6" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:49.554044 25265 raft_consensus.cc:2454] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:49.554358 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0d3af4df9b410baaa088c744eeb2a6" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:49.554445 24938 leader_election.cc:302] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 74f9aa91c2c448f980b512fe2e8d672e, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:49.554489 22717 raft_consensus.cc:2454] T cf0d3af4df9b410baaa088c744eeb2a6 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 0.
I20240502 07:58:49.554596 26696 raft_consensus.cc:2781] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:49.554653 26696 raft_consensus.cc:482] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:49.554697 26696 raft_consensus.cc:3037] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:49.555634 26696 raft_consensus.cc:504] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:49.555929 26696 leader_election.cc:288] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:49.556288 27339 raft_consensus.cc:482] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:49.556373 27339 raft_consensus.cc:504] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:49.556387 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0d3af4df9b410baaa088c744eeb2a6" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:49.556492 22717 raft_consensus.cc:3037] T cf0d3af4df9b410baaa088c744eeb2a6 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:49.557036 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0d3af4df9b410baaa088c744eeb2a6" candidate_uuid: "74f9aa91c2c448f980b512fe2e8d672e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:49.557142 25265 raft_consensus.cc:3037] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:49.557822 22717 raft_consensus.cc:2454] T cf0d3af4df9b410baaa088c744eeb2a6 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:49.558050 24936 leader_election.cc:302] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:49.558961 25265 raft_consensus.cc:2454] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 74f9aa91c2c448f980b512fe2e8d672e in term 1.
I20240502 07:58:49.559099 24871 raft_consensus.cc:1203] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:49.561527 27339 leader_election.cc:288] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:49.561923 26696 raft_consensus.cc:2781] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:49.561993 26696 raft_consensus.cc:686] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEADER]: Becoming Leader. State: Replica: 74f9aa91c2c448f980b512fe2e8d672e, State: Running, Role: LEADER
I20240502 07:58:49.562093 26696 consensus_queue.cc:237] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:49.562352 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c41dd9727c3f44cbae17bad891e55636" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:49.562458 22845 raft_consensus.cc:2454] T c41dd9727c3f44cbae17bad891e55636 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:49.562959 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c41dd9727c3f44cbae17bad891e55636" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:49.563043 25265 raft_consensus.cc:2454] T c41dd9727c3f44cbae17bad891e55636 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:49.563136 24808 leader_election.cc:302] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:49.564606 22435 catalog_manager.cc:5484] T cf0d3af4df9b410baaa088c744eeb2a6 P 74f9aa91c2c448f980b512fe2e8d672e reported cstate change: term changed from 0 to 1, leader changed from <none> to 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134). New cstate: current_term: 1 leader_uuid: "74f9aa91c2c448f980b512fe2e8d672e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:49.565281 27339 raft_consensus.cc:2781] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:49.565330 27339 raft_consensus.cc:482] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:49.565361 27339 raft_consensus.cc:3037] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:49.566160 27339 raft_consensus.cc:504] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:49.566426 27339 leader_election.cc:288] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:49.566722 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c41dd9727c3f44cbae17bad891e55636" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:49.566810 22845 raft_consensus.cc:3037] T c41dd9727c3f44cbae17bad891e55636 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:49.567628 22845 raft_consensus.cc:2454] T c41dd9727c3f44cbae17bad891e55636 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:49.567942 24808 leader_election.cc:302] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:49.568123 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c41dd9727c3f44cbae17bad891e55636" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:49.568205 25265 raft_consensus.cc:3037] T c41dd9727c3f44cbae17bad891e55636 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:49.568532 27339 raft_consensus.cc:2781] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:49.568612 27339 raft_consensus.cc:686] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:58:49.568782 27339 consensus_queue.cc:237] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:49.568997 25265 raft_consensus.cc:2454] T c41dd9727c3f44cbae17bad891e55636 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:49.570573 22435 catalog_manager.cc:5484] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:49.581542 27495 raft_consensus.cc:482] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:49.581625 27495 raft_consensus.cc:504] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.581925 27495 leader_election.cc:288] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:49.582227 25003 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "73dd5ed5e23b44839c956ffa01d69345" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e" is_pre_election: true
I20240502 07:58:49.582329 25003 raft_consensus.cc:2454] T 73dd5ed5e23b44839c956ffa01d69345 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:49.582343 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "73dd5ed5e23b44839c956ffa01d69345" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:49.582449 24871 raft_consensus.cc:2454] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:49.582643 25201 leader_election.cc:302] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 74f9aa91c2c448f980b512fe2e8d672e, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:49.582850 27495 raft_consensus.cc:2781] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:49.582890 27495 raft_consensus.cc:482] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:49.582919 27495 raft_consensus.cc:3037] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:49.583806 27495 raft_consensus.cc:504] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.584096 27495 leader_election.cc:288] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:49.584335 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "73dd5ed5e23b44839c956ffa01d69345" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:49.584429 24871 raft_consensus.cc:3037] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:49.584481 25003 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "73dd5ed5e23b44839c956ffa01d69345" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e"
I20240502 07:58:49.584565 25003 raft_consensus.cc:3037] T 73dd5ed5e23b44839c956ffa01d69345 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:49.584692 22717 consensus_queue.cc:237] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:49.585355 24871 raft_consensus.cc:2454] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:49.585428 25003 raft_consensus.cc:2454] T 73dd5ed5e23b44839c956ffa01d69345 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:49.585821 25199 leader_election.cc:302] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
W20240502 07:58:49.586242 22653 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): Couldn't send request to peer 4654231dfe994c1a9d2e37d9c3d75699. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 539cf5d4596f481cb03ab4d13a19198c. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:49.586457 22845 raft_consensus.cc:1261] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 07:58:49.586846 28450 consensus_queue.cc:1035] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:49.586927 25003 raft_consensus.cc:1261] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 07:58:49.587137 27495 raft_consensus.cc:2781] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:49.587205 27495 raft_consensus.cc:686] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:49.587217 28450 consensus_queue.cc:1035] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:49.587293 27495 consensus_queue.cc:237] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.588984 22435 catalog_manager.cc:5484] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:49.590049 27303 raft_consensus.cc:2932] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:49.590430 22845 raft_consensus.cc:2932] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:49.594219 22435 catalog_manager.cc:5484] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 6 to 7, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:49.594202 25003 raft_consensus.cc:2932] T 539cf5d4596f481cb03ab4d13a19198c P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 07:58:49.648427 25003 raft_consensus.cc:1203] T bd1c2abfe2a3416bb10d6c14d3fed46a P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 07:58:49.656726 25670 raft_consensus.cc:482] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:49.656823 25670 raft_consensus.cc:504] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.657143 25670 leader_election.cc:288] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:49.657472 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4679a4f9194a4dbcb85c28e31f6ac894" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:49.657616 24871 raft_consensus.cc:2454] T 4679a4f9194a4dbcb85c28e31f6ac894 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:49.657755 25003 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4679a4f9194a4dbcb85c28e31f6ac894" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e" is_pre_election: true
I20240502 07:58:49.657873 25003 raft_consensus.cc:2454] T 4679a4f9194a4dbcb85c28e31f6ac894 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:49.658092 22914 leader_election.cc:302] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 74f9aa91c2c448f980b512fe2e8d672e; no voters:
I20240502 07:58:49.658258 25670 raft_consensus.cc:2781] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:49.658316 25670 raft_consensus.cc:482] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:49.658360 25670 raft_consensus.cc:3037] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:49.659288 25670 raft_consensus.cc:504] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.659562 25670 leader_election.cc:288] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:49.659780 25003 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4679a4f9194a4dbcb85c28e31f6ac894" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "74f9aa91c2c448f980b512fe2e8d672e"
I20240502 07:58:49.659818 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4679a4f9194a4dbcb85c28e31f6ac894" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:49.659911 24871 raft_consensus.cc:3037] T 4679a4f9194a4dbcb85c28e31f6ac894 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:49.660727 24871 raft_consensus.cc:2454] T 4679a4f9194a4dbcb85c28e31f6ac894 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:49.660940 22912 leader_election.cc:302] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:49.661121 25670 raft_consensus.cc:2781] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:49.661195 25670 raft_consensus.cc:686] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:49.659904 25003 raft_consensus.cc:3037] T 4679a4f9194a4dbcb85c28e31f6ac894 P 74f9aa91c2c448f980b512fe2e8d672e [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:49.661319 25670 consensus_queue.cc:237] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:49.662168 25003 raft_consensus.cc:2454] T 4679a4f9194a4dbcb85c28e31f6ac894 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:49.663246 22442 catalog_manager.cc:5484] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:49.666285 28989 ts_tablet_manager.cc:902] T 68f96ca3354645fd91f666109eeaf847 P 74f9aa91c2c448f980b512fe2e8d672e: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
F20240502 07:58:49.666452 28989 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-1e370c8fe68b4392ba3720533e538de3.queue_time_us, tablet-download-pool-1e370c8fe68b4392ba3720533e538de3.queue_time_us) (tablet-download-pool-1e370c8fe68b4392ba3720533e538de3.run_cpu_time_us, tablet-download-pool-1e370c8fe68b4392ba3720533e538de3.run_cpu_time_us) (tablet-download-pool-1e370c8fe68b4392ba3720533e538de3.run_wall_time_us, tablet-download-pool-1e370c8fe68b4392ba3720533e538de3.run_wall_time_us) (tablet-download-pool-539cf5d4596f481cb03ab4d13a19198c.queue_time_us, tablet-download-pool-539cf5d4596f481cb03ab4d13a19198c.queue_time_us) (tablet-download-pool-539cf5d4596f481cb03ab4d13a19198c.run_cpu_time_us, tablet-download-pool-539cf5d4596f481cb03ab4d13a19198c.run_cpu_time_us) (tablet-download-pool-539cf5d4596f481cb03ab4d13a19198c.run_wall_time_us, tablet-download-pool-539cf5d4596f481cb03ab4d13a19198c.run_wall_time_us) (tablet-download-pool-68f96ca3354645fd91f666109eeaf847.queue_time_us, tablet-download-pool-68f96ca3354645fd91f666109eeaf847.queue_time_us) (tablet-download-pool-6dbf36c0f044466ab13a53bfe0d6cb53.queue_time_us, tablet-download-pool-6dbf36c0f044466ab13a53bfe0d6cb53.queue_time_us) (tablet-download-pool-6dbf36c0f044466ab13a53bfe0d6cb53.run_cpu_time_us, tablet-download-pool-6dbf36c0f044466ab13a53bfe0d6cb53.run_cpu_time_us) (tablet-download-pool-6dbf36c0f044466ab13a53bfe0d6cb53.run_wall_time_us, tablet-download-pool-6dbf36c0f044466ab13a53bfe0d6cb53.run_wall_time_us) (tablet-download-pool-a5374313bbba4b10904e0ea18851c07d.queue_time_us, tablet-download-pool-a5374313bbba4b10904e0ea18851c07d.queue_time_us) (tablet-download-pool-a5374313bbba4b10904e0ea18851c07d.run_cpu_time_us, tablet-download-pool-a5374313bbba4b10904e0ea18851c07d.run_cpu_time_us) (tablet-download-pool-a5374313bbba4b10904e0ea18851c07d.run_wall_time_us, tablet-download-pool-a5374313bbba4b10904e0ea18851c07d.run_wall_time_us) (tablet-download-pool-a8dcc36740be4ca3ac1d43591f7549bd.queue_time_us, tablet-download-pool-a8dcc36740be4ca3ac1d43591f7549bd.queue_time_us) (tablet-download-pool-a8dcc36740be4ca3ac1d43591f7549bd.run_cpu_time_us, tablet-download-pool-a8dcc36740be4ca3ac1d43591f7549bd.run_cpu_time_us) (tablet-download-pool-a8dcc36740be4ca3ac1d43591f7549bd.run_wall_time_us, tablet-download-pool-a8dcc36740be4ca3ac1d43591f7549bd.run_wall_time_us) (tablet-download-pool-b583c8775a874ad1a89c82c00c05bbbd.queue_time_us, tablet-download-pool-b583c8775a874ad1a89c82c00c05bbbd.queue_time_us) (tablet-download-pool-b583c8775a874ad1a89c82c00c05bbbd.run_cpu_time_us, tablet-download-pool-b583c8775a874ad1a89c82c00c05bbbd.run_cpu_time_us) (tablet-download-pool-b583c8775a874ad1a89c82c00c05bbbd.run_wall_time_us, tablet-download-pool-b583c8775a874ad1a89c82c00c05bbbd.run_wall_time_us) (tablet-download-pool-bd1c2abfe2a3416bb10d6c14d3fed46a.queue_time_us, tablet-download-pool-bd1c2abfe2a3416bb10d6c14d3fed46a.queue_time_us) (tablet-download-pool-bd1c2abfe2a3416bb10d6c14d3fed46a.run_cpu_time_us, tablet-download-pool-bd1c2abfe2a3416bb10d6c14d3fed46a.run_cpu_time_us) (tablet-download-pool-bd1c2abfe2a3416bb10d6c14d3fed46a.run_wall_time_us, tablet-download-pool-bd1c2abfe2a3416bb10d6c14d3fed46a.run_wall_time_us) (tablet-download-pool-c3a7b2e52ba641e993cf1d5dc9724473.queue_time_us, tablet-download-pool-c3a7b2e52ba641e993cf1d5dc9724473.queue_time_us) (tablet-download-pool-c3a7b2e52ba641e993cf1d5dc9724473.run_cpu_time_us, tablet-download-pool-c3a7b2e52ba641e993cf1d5dc9724473.run_cpu_time_us) (tablet-download-pool-c3a7b2e52ba641e993cf1d5dc9724473.run_wall_time_us, tablet-download-pool-c3a7b2e52ba641e993cf1d5dc9724473.run_wall_time_us) (tablet-download-pool-c8c7521bb8414364b927eed84be37621.queue_time_us, tablet-download-pool-c8c7521bb8414364b927eed84be37621.queue_time_us) (tablet-download-pool-c8c7521bb8414364b927eed84be37621.run_cpu_time_us, tablet-download-pool-c8c7521bb8414364b927eed84be37621.run_cpu_time_us) (tablet-download-pool-c8c7521bb8414364b927eed84be37621.run_wall_time_us, tablet-download-pool-c8c7521bb8414364b927eed84be37621.run_wall_time_us) (tablet-download-pool-cceaa8887081401dbc9154d265004c7b.queue_time_us, tablet-download-pool-cceaa8887081401dbc9154d265004c7b.queue_time_us) (tablet-download-pool-cceaa8887081401dbc9154d265004c7b.run_cpu_time_us, tablet-download-pool-cceaa8887081401dbc9154d265004c7b.run_cpu_time_us) (tablet-download-pool-cceaa8887081401dbc9154d265004c7b.run_wall_time_us, tablet-download-pool-cceaa8887081401dbc9154d265004c7b.run_wall_time_us) (tablet-download-pool-da39029564d042f6acfc23f99fb63dfd.queue_time_us, tablet-download-pool-da39029564d042f6acfc23f99fb63dfd.queue_time_us) (tablet-download-pool-da39029564d042f6acfc23f99fb63dfd.run_cpu_time_us, tablet-download-pool-da39029564d042f6acfc23f99fb63dfd.run_cpu_time_us) (tablet-download-pool-da39029564d042f6acfc23f99fb63dfd.run_wall_time_us, tablet-download-pool-da39029564d042f6acfc23f99fb63dfd.run_wall_time_us) (tablet-download-pool-e25ba5cc77334a3595a4575fa98476f0.queue_time_us, tablet-download-pool-e25ba5cc77334a3595a4575fa98476f0.queue_time_us) (tablet-download-pool-e25ba5cc77334a3595a4575fa98476f0.run_cpu_time_us, tablet-download-pool-e25ba5cc77334a3595a4575fa98476f0.run_cpu_time_us) (tablet-download-pool-e25ba5cc77334a3595a4575fa98476f0.run_wall_time_us, tablet-download-pool-e25ba5cc77334a3595a4575fa98476f0.run_wall_time_us) (tablet-download-pool-eb0a7b1308114769b556795ae18a746e.queue_time_us, tablet-download-pool-eb0a7b1308114769b556795ae18a746e.queue_time_us) (tablet-download-pool-eb0a7b1308114769b556795ae18a746e.run_cpu_time_us, tablet-download-pool-eb0a7b1308114769b556795ae18a746e.run_cpu_time_us) (tablet-download-pool-eb0a7b1308114769b556795ae18a746e.run_wall_time_us, tablet-download-pool-eb0a7b1308114769b556795ae18a746e.run_wall_time_us) (tablet-download-pool-f1fa312fe8e44d23970c5e3ba98c2c58.queue_time_us, tablet-download-pool-f1fa312fe8e44d23970c5e3ba98c2c58.queue_time_us) (tablet-download-pool-f1fa312fe8e44d23970c5e3ba98c2c58.run_cpu_time_us, tablet-download-pool-f1fa312fe8e44d23970c5e3ba98c2c58.run_cpu_time_us) (tablet-download-pool-f1fa312fe8e44d23970c5e3ba98c2c58.run_wall_time_us, tablet-download-pool-f1fa312fe8e44d23970c5e3ba98c2c58.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7f7b32b9adcd google::LogMessage::Fail() at ??:0
@ 0x7f7b32b9eb93 google::LogMessage::SendToLog() at ??:0
@ 0x7f7b32b9a7cc google::LogMessage::Flush() at ??:0
@ 0x7f7b32b9bf59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f7b34004044 kudu::TraceMetrics::InternName() at ??:0
I20240502 07:58:49.680970 25134 raft_consensus.cc:1203] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
@ 0x7f7b33fe8f98 kudu::ThreadPool::ThreadPool() at ??:0
I20240502 07:58:49.683991 27488 raft_consensus.cc:1050] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699: attempting to promote NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f to VOTER
I20240502 07:58:49.684403 27488 consensus_queue.cc:237] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } }
@ 0x7f7b33fe6efc kudu::ThreadPoolBuilder::Build() at ??:0
I20240502 07:58:49.685725 25003 raft_consensus.cc:1261] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:49.685745 22584 raft_consensus.cc:1261] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:49.685796 22978 raft_consensus.cc:1261] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:49.686132 28757 consensus_queue.cc:1035] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:49.686689 28643 consensus_queue.cc:1035] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:49.687678 29193 consensus_queue.cc:1035] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
@ 0x7f7b36b9977a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
I20240502 07:58:49.689069 28757 raft_consensus.cc:2932] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } } }
I20240502 07:58:49.690797 25003 raft_consensus.cc:2932] T 5487e66cb4d8416094a1321cb989f018 P 74f9aa91c2c448f980b512fe2e8d672e [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } } }
@ 0x7f7b36ba2a45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
I20240502 07:58:49.692235 22978 raft_consensus.cc:2932] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } } }
I20240502 07:58:49.692262 22584 raft_consensus.cc:2932] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } } }
I20240502 07:58:49.693397 22435 catalog_manager.cc:5484] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index 2 to 3, 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
@ 0x7f7b36c2376f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
I20240502 07:58:49.698241 28470 raft_consensus.cc:1050] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3: attempting to promote NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e to VOTER
I20240502 07:58:49.698737 28470 consensus_queue.cc:237] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } }
@ 0x7f7b36c374f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
I20240502 07:58:49.700500 25003 raft_consensus.cc:1261] T da39029564d042f6acfc23f99fb63dfd P 74f9aa91c2c448f980b512fe2e8d672e [term 1 LEARNER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:49.700891 25134 raft_consensus.cc:1261] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:49.700894 27303 consensus_queue.cc:1035] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:49.701457 27303 consensus_queue.cc:1035] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
@ 0x7f7b36c22226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
I20240502 07:58:49.704887 22978 raft_consensus.cc:1261] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:49.705052 25134 consensus_queue.cc:237] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } }
I20240502 07:58:49.705230 27303 consensus_queue.cc:1035] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
@ 0x7f7b36c31c0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55c36c43675c std::function<>::operator()() at ??:0
I20240502 07:58:49.706460 22979 raft_consensus.cc:1261] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:49.706611 22584 raft_consensus.cc:1261] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:49.706817 28643 consensus_queue.cc:1035] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:49.706964 27488 consensus_queue.cc:1035] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:49.708459 27303 raft_consensus.cc:2932] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
@ 0x7f7b33febe06 kudu::ThreadPool::DispatchThread() at ??:0
I20240502 07:58:49.710256 28643 raft_consensus.cc:2932] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } } }
@ 0x7f7b33fec6ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
I20240502 07:58:49.712074 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 5487e66cb4d8416094a1321cb989f018 with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:49.712481 22978 raft_consensus.cc:2932] T 5487e66cb4d8416094a1321cb989f018 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } } }
I20240502 07:58:49.712783 22979 raft_consensus.cc:2932] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
@ 0x7f7b33fee03b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55c36c43675c std::function<>::operator()() at ??:0
@ 0x7f7b33fdb807 kudu::Thread::SuperviseThread() at ??:0
I20240502 07:58:49.719126 22444 catalog_manager.cc:5484] T 5487e66cb4d8416094a1321cb989f018 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index 3 to 4, VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
@ 0x7f7b348196db start_thread at ??:0
I20240502 07:58:49.719802 25134 raft_consensus.cc:2932] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
@ 0x7f7b31d5171f clone at ??:0
I20240502 07:58:49.722229 24983 tablet_service.cc:1514] Processing DeleteTablet for tablet 5487e66cb4d8416094a1321cb989f018 with delete_type TABLET_DATA_TOMBSTONED (TS 74f9aa91c2c448f980b512fe2e8d672e not found in new config with opid_index 4) from {username='slave'} at 127.0.0.1:42878
I20240502 07:58:49.723310 22440 catalog_manager.cc:5484] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index 2 to 3, 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } }
I20240502 07:58:49.723703 22584 raft_consensus.cc:2932] T 5487e66cb4d8416094a1321cb989f018 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: false } } }
I20240502 07:58:49.745029 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:49.748605 24871 raft_consensus.cc:1261] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:49.749611 28205 consensus_queue.cc:1035] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:49.752586 22444 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet 73dd5ed5e23b44839c956ffa01d69345 (table rebalancer_dynamic_table [id=f01f3d0fc0ba4997b40287d9e08e8fc3]). Expected version 1 got 0
I20240502 07:58:49.752806 29207 tablet.cc:1722] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:49.753109 29207 tablet.cc:1620] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:49.754093 22442 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet 73dd5ed5e23b44839c956ffa01d69345 (table rebalancer_dynamic_table [id=f01f3d0fc0ba4997b40287d9e08e8fc3]). Expected version 1 got 0
I20240502 07:58:49.754109 29208 tablet.cc:1722] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:49.754375 29208 tablet.cc:1620] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:49.754909 22444 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet 73dd5ed5e23b44839c956ffa01d69345 (table rebalancer_dynamic_table [id=f01f3d0fc0ba4997b40287d9e08e8fc3]). Expected version 1 got 0
I20240502 07:58:49.757864 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=f01f3d0fc0ba4997b40287d9e08e8fc3] alter complete (version 1)
I20240502 07:58:49.773216 29007 ts_tablet_manager.cc:902] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:49.773684 29007 tablet_copy_client.cc:308] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33387
I20240502 07:58:49.774070 22737 tablet_copy_service.cc:140] P 725969d490a04fdfb3ba890c3f5096e3: Received BeginTabletCopySession request for tablet 9d2283e5d28347519b38e876d840608e from peer 7eced220cf1642f3873fc385b6191e69 ({username='slave'} at 127.16.85.136:54683)
I20240502 07:58:49.774145 22737 tablet_copy_service.cc:161] P 725969d490a04fdfb3ba890c3f5096e3: Beginning new tablet copy session on tablet 9d2283e5d28347519b38e876d840608e from peer 7eced220cf1642f3873fc385b6191e69 at {username='slave'} at 127.16.85.136:54683: session id = 7eced220cf1642f3873fc385b6191e69-9d2283e5d28347519b38e876d840608e
I20240502 07:58:49.775034 22737 tablet_copy_source_session.cc:205] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:49.776089 29007 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9d2283e5d28347519b38e876d840608e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:49.778052 25265 raft_consensus.cc:1203] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 07:58:49.778636 29007 tablet_copy_client.cc:783] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:49.778777 29007 tablet_copy_client.cc:647] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:49.780756 29007 tablet_copy_client.cc:523] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:49.781898 29007 tablet_bootstrap.cc:492] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:49.808813 29007 tablet_bootstrap.cc:492] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:49.809170 29007 tablet_bootstrap.cc:492] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69: Bootstrap complete.
I20240502 07:58:49.809326 29007 ts_tablet_manager.cc:1367] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.027s user 0.027s sys 0.000s
I20240502 07:58:49.809731 29007 raft_consensus.cc:348] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69 [term 2 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:49.809852 29007 raft_consensus.cc:729] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69 [term 2 LEARNER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: LEARNER
I20240502 07:58:49.809933 29007 consensus_queue.cc:260] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:49.810173 29007 ts_tablet_manager.cc:1398] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:49.811326 22737 tablet_copy_service.cc:342] P 725969d490a04fdfb3ba890c3f5096e3: Request end of tablet copy session 7eced220cf1642f3873fc385b6191e69-9d2283e5d28347519b38e876d840608e received from {username='slave'} at 127.16.85.136:54683
I20240502 07:58:49.811417 22737 tablet_copy_service.cc:433] P 725969d490a04fdfb3ba890c3f5096e3: ending tablet copy session 7eced220cf1642f3873fc385b6191e69-9d2283e5d28347519b38e876d840608e on tablet 9d2283e5d28347519b38e876d840608e with peer 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:49.861315 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:49.866397 29207 tablet.cc:1722] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:49.866696 29207 tablet.cc:1620] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:49.867583 29208 tablet.cc:1722] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:49.867882 29208 tablet.cc:1620] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:49.872375 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=f01f3d0fc0ba4997b40287d9e08e8fc3] alter complete (version 2)
W20240502 07:58:49.942531 25201 connection.cc:535] client connection to 127.16.85.134:34857 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 07:58:49.942587 22783 connection.cc:535] client connection to 127.16.85.134:34857 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 07:58:49.942693 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 07:58:49.942715 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 07:58:49.942931 22914 connection.cc:535] client connection to 127.16.85.134:34857 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 07:58:49.943002 22430 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.16.85.134:34857 (error 108)
W20240502 07:58:49.943025 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 07:58:49.943248 22652 connection.cc:535] client connection to 127.16.85.134:34857 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 07:58:49.943361 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 07:58:49.943802 22521 connection.cc:535] client connection to 127.16.85.134:34857 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 07:58:49.943922 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 07:58:49.944865 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 07:58:49.944936 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 07:58:49.945181 25201 consensus_peers.cc:485] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 07:58:49.945240 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 07:58:49.945281 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 5487e66cb4d8416094a1321cb989f018: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:58:49.945570 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:58:49.945920 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 07:58:49.945983 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 07:58:49.946046 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 07:58:49.946050 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 07:58:49.967845 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:49.968195 28823 consensus_queue.cc:1035] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:58:49.968602 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:49.971717 28823 consensus_queue.cc:1035] T c41dd9727c3f44cbae17bad891e55636 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:49.975443 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:49.980420 29207 tablet.cc:1722] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:49.980947 29208 tablet.cc:1722] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
W20240502 07:58:49.985880 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 07:58:49.986209 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:49.986250 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=f01f3d0fc0ba4997b40287d9e08e8fc3] alter complete (version 3)
I20240502 07:58:50.007598 22584 consensus_queue.cc:237] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:50.008754 25134 raft_consensus.cc:1261] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 07:58:50.008850 25265 raft_consensus.cc:1261] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 07:58:50.009181 27136 consensus_queue.cc:1035] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:50.009338 24714 consensus_queue.cc:1035] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:50.010828 28188 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.011229 25265 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.011555 25134 raft_consensus.cc:2932] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.012986 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet dd815609d7324cc588f02b6cb40cfbfd with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:50.013978 29022 raft_consensus.cc:1050] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 6d02017318a2461796d431962fe6a537 to VOTER
I20240502 07:58:50.014081 22442 catalog_manager.cc:5484] T dd815609d7324cc588f02b6cb40cfbfd P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index 8 to 9, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.014382 29022 consensus_queue.cc:237] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:50.015152 26995 raft_consensus.cc:1050] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3: attempting to promote NON_VOTER 6d02017318a2461796d431962fe6a537 to VOTER
I20240502 07:58:50.015509 26995 consensus_queue.cc:237] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:50.016242 24872 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:50.016585 25670 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:50.016901 24872 raft_consensus.cc:1261] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:50.017096 25265 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:50.017501 22845 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:50.017479 27303 consensus_queue.cc:1035] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:50.017602 22847 raft_consensus.cc:1261] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:50.017901 22584 raft_consensus.cc:1261] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 07:58:50.017971 27303 consensus_queue.cc:1035] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:50.018224 25670 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:50.018605 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet dd815609d7324cc588f02b6cb40cfbfd with delete_type TABLET_DATA_TOMBSTONED (TS 7db4178eed174225bfb24f5f9445ce88 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:50.018949 29337 tablet_replica.cc:330] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:50.019080 29337 raft_consensus.cc:2229] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:50.019277 29337 raft_consensus.cc:2258] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:50.019289 27333 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:50.019388 27303 consensus_queue.cc:1035] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:50.020107 29337 ts_tablet_manager.cc:1875] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:50.021387 27333 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:50.021715 28470 raft_consensus.cc:2932] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:50.022637 29337 ts_tablet_manager.cc:1888] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 07:58:50.022850 29337 log.cc:1196] T dd815609d7324cc588f02b6cb40cfbfd P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/dd815609d7324cc588f02b6cb40cfbfd
I20240502 07:58:50.022931 24872 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:50.023636 22845 raft_consensus.cc:2932] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:50.023996 22444 catalog_manager.cc:5484] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 7 to 8, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:50.025338 22440 catalog_manager.cc:5484] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:50.025861 24872 raft_consensus.cc:2932] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:50.027379 22845 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:50.027784 25265 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:50.029186 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet dd815609d7324cc588f02b6cb40cfbfd (table rebalance_test_table_17 [id=35a66cc6bd364d818b35bceada233a98]) successfully deleted
I20240502 07:58:50.031955 22584 raft_consensus.cc:2932] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6d02017318a2461796d431962fe6a537 (127.16.85.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:50.034672 22717 consensus_queue.cc:237] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:50.035549 22584 raft_consensus.cc:1261] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:50.035567 24871 raft_consensus.cc:1261] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 07:58:50.035866 27582 consensus_queue.cc:1035] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:50.035997 28450 consensus_queue.cc:1035] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 07:58:50.037451 28450 raft_consensus.cc:2932] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:50.037884 24871 raft_consensus.cc:2932] T 023958ce03b84b2da063c13e4284e04d P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:50.038332 22584 raft_consensus.cc:2932] T 023958ce03b84b2da063c13e4284e04d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:50.039758 22428 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 023958ce03b84b2da063c13e4284e04d with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:50.040547 22435 catalog_manager.cc:5484] T 023958ce03b84b2da063c13e4284e04d P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 5 to 6, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:50.043838 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet 023958ce03b84b2da063c13e4284e04d with delete_type TABLET_DATA_TOMBSTONED (TS 7db4178eed174225bfb24f5f9445ce88 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:50.044036 29337 tablet_replica.cc:330] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:50.044158 29337 raft_consensus.cc:2229] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:50.044292 29337 raft_consensus.cc:2258] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:50.045074 29337 ts_tablet_manager.cc:1875] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:50.046926 29337 ts_tablet_manager.cc:1888] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 07:58:50.046998 29337 log.cc:1196] T 023958ce03b84b2da063c13e4284e04d P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/023958ce03b84b2da063c13e4284e04d
I20240502 07:58:50.047453 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 023958ce03b84b2da063c13e4284e04d (table rebalance_test_table_19 [id=65deae5d69b44af784a28c39cd113241]) successfully deleted
I20240502 07:58:50.062031 29368 ts_tablet_manager.cc:902] T 539cf5d4596f481cb03ab4d13a19198c P 4654231dfe994c1a9d2e37d9c3d75699: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:50.062541 29368 tablet_copy_client.cc:308] T 539cf5d4596f481cb03ab4d13a19198c P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33387
I20240502 07:58:50.062943 22737 tablet_copy_service.cc:140] P 725969d490a04fdfb3ba890c3f5096e3: Received BeginTabletCopySession request for tablet 539cf5d4596f481cb03ab4d13a19198c from peer 4654231dfe994c1a9d2e37d9c3d75699 ({username='slave'} at 127.16.85.135:38693)
I20240502 07:58:50.063046 22737 tablet_copy_service.cc:161] P 725969d490a04fdfb3ba890c3f5096e3: Beginning new tablet copy session on tablet 539cf5d4596f481cb03ab4d13a19198c from peer 4654231dfe994c1a9d2e37d9c3d75699 at {username='slave'} at 127.16.85.135:38693: session id = 4654231dfe994c1a9d2e37d9c3d75699-539cf5d4596f481cb03ab4d13a19198c
I20240502 07:58:50.063791 22737 tablet_copy_source_session.cc:205] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:50.064730 29368 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 539cf5d4596f481cb03ab4d13a19198c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.066663 25265 raft_consensus.cc:1203] T a8dcc36740be4ca3ac1d43591f7549bd P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 07:58:50.066884 29368 tablet_copy_client.cc:783] T 539cf5d4596f481cb03ab4d13a19198c P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:50.067014 29368 tablet_copy_client.cc:647] T 539cf5d4596f481cb03ab4d13a19198c P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:50.068209 29022 raft_consensus.cc:1050] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 to VOTER
I20240502 07:58:50.068272 22979 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "ca85b40df34d4e1c91a55d7dd69e1494"
dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:46644
I20240502 07:58:50.068363 22979 raft_consensus.cc:595] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:50.068603 29022 consensus_queue.cc:237] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:50.069867 25134 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:50.070142 29368 tablet_copy_client.cc:523] T 539cf5d4596f481cb03ab4d13a19198c P 4654231dfe994c1a9d2e37d9c3d75699: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:50.071373 29368 tablet_bootstrap.cc:492] T 539cf5d4596f481cb03ab4d13a19198c P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
W20240502 07:58:50.072932 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:50.073060 22845 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:50.073436 29334 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:50.074023 27333 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:50.077714 29334 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:50.080190 25134 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:50.080435 22845 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:50.080497 22442 catalog_manager.cc:5484] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 7 to 8, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:50.088193 22444 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:50.088299 22444 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:50.088781 22979 consensus_queue.cc:237] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 07:58:50.090031 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 73dd5ed5e23b44839c956ffa01d69345
I20240502 07:58:50.090363 22584 raft_consensus.cc:1261] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:50.090397 24871 raft_consensus.cc:1261] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:50.090770 29022 consensus_queue.cc:1035] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:50.091023 25235 tablet_service.cc:1514] Processing DeleteTablet for tablet 73dd5ed5e23b44839c956ffa01d69345 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:50 UTC) from {username='slave'} at 127.0.0.1:33118
W20240502 07:58:50.091272 22915 consensus_peers.cc:485] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): Couldn't send request to peer 7db4178eed174225bfb24f5f9445ce88. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6af67c65672f45a68e960936351e223f. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:50.091538 29022 consensus_queue.cc:1035] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:50.091868 24850 tablet_service.cc:1514] Processing DeleteTablet for tablet 73dd5ed5e23b44839c956ffa01d69345 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:50 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:50.092244 29395 tablet_replica.cc:330] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:50.092394 29395 raft_consensus.cc:2229] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:50.092512 29395 raft_consensus.cc:2258] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:50.092909 24871 raft_consensus.cc:2932] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:50.092594 27333 raft_consensus.cc:2932] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:50.093125 29395 ts_tablet_manager.cc:1875] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:50.095573 22442 catalog_manager.cc:5484] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537 reported cstate change: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 07:58:50.096395 29395 ts_tablet_manager.cc:1888] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:50.096469 29395 log.cc:1196] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/73dd5ed5e23b44839c956ffa01d69345
I20240502 07:58:50.098105 29395 ts_tablet_manager.cc:1909] T 73dd5ed5e23b44839c956ffa01d69345 P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:50.098767 29393 tablet_replica.cc:330] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:50.098876 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet 73dd5ed5e23b44839c956ffa01d69345 (table rebalancer_dynamic_table_new_name [id=f01f3d0fc0ba4997b40287d9e08e8fc3]) successfully deleted
I20240502 07:58:50.099004 29393 raft_consensus.cc:2229] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:50.099534 29393 raft_consensus.cc:2258] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:50.100127 29393 ts_tablet_manager.cc:1875] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:50.100065 22584 raft_consensus.cc:2932] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
W20240502 07:58:50.101016 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:50.102592 25670 consensus_queue.cc:1035] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:50.103657 29368 tablet_bootstrap.cc:492] T 539cf5d4596f481cb03ab4d13a19198c P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:50.103919 29368 tablet_bootstrap.cc:492] T 539cf5d4596f481cb03ab4d13a19198c P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap complete.
I20240502 07:58:50.104039 29368 ts_tablet_manager.cc:1367] T 539cf5d4596f481cb03ab4d13a19198c P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.033s user 0.023s sys 0.004s
I20240502 07:58:50.104440 29368 raft_consensus.cc:348] T 539cf5d4596f481cb03ab4d13a19198c P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:50.104558 29368 raft_consensus.cc:729] T 539cf5d4596f481cb03ab4d13a19198c P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: LEARNER
I20240502 07:58:50.104686 29368 consensus_queue.cc:260] T 539cf5d4596f481cb03ab4d13a19198c P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 07:58:50.105178 29393 ts_tablet_manager.cc:1888] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:50.105288 29393 log.cc:1196] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/73dd5ed5e23b44839c956ffa01d69345
I20240502 07:58:50.105583 29368 ts_tablet_manager.cc:1398] T 539cf5d4596f481cb03ab4d13a19198c P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:50.105978 22737 tablet_copy_service.cc:342] P 725969d490a04fdfb3ba890c3f5096e3: Request end of tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-539cf5d4596f481cb03ab4d13a19198c received from {username='slave'} at 127.16.85.135:38693
I20240502 07:58:50.106053 22737 tablet_copy_service.cc:433] P 725969d490a04fdfb3ba890c3f5096e3: ending tablet copy session 4654231dfe994c1a9d2e37d9c3d75699-539cf5d4596f481cb03ab4d13a19198c on tablet 539cf5d4596f481cb03ab4d13a19198c with peer 4654231dfe994c1a9d2e37d9c3d75699
I20240502 07:58:50.107993 29393 ts_tablet_manager.cc:1909] T 73dd5ed5e23b44839c956ffa01d69345 P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
W20240502 07:58:50.108223 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:50.108803 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet 73dd5ed5e23b44839c956ffa01d69345 (table rebalancer_dynamic_table_new_name [id=f01f3d0fc0ba4997b40287d9e08e8fc3]) successfully deleted
W20240502 07:58:50.116628 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 07:58:50.120520 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 07:58:50.136886 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:50.156862 28188 raft_consensus.cc:1050] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85: attempting to promote NON_VOTER 7eced220cf1642f3873fc385b6191e69 to VOTER
I20240502 07:58:50.157296 28188 consensus_queue.cc:237] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:50.158599 25265 raft_consensus.cc:1261] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:50.158645 24871 raft_consensus.cc:1261] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:50.158927 22717 raft_consensus.cc:1261] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:50.158991 28188 consensus_queue.cc:1035] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:50.159267 28188 consensus_queue.cc:1035] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:50.159729 28188 consensus_queue.cc:1035] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:50.161288 27136 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.161675 25265 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.164520 22444 catalog_manager.cc:5484] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:50.164551 24871 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.165067 22717 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.166500 22979 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "ca85b40df34d4e1c91a55d7dd69e1494"
dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:46704
I20240502 07:58:50.166605 22979 raft_consensus.cc:595] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:50.168536 22584 consensus_queue.cc:237] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:50.169597 25266 raft_consensus.cc:1261] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 07:58:50.169924 29455 consensus_queue.cc:1035] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:50.170683 24871 raft_consensus.cc:1261] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 07:58:50.171105 29455 consensus_queue.cc:1035] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:50.172430 29455 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.172839 25265 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.174293 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9e5fe86b44744df9b2e4d430fdec716f with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:50.174257 24871 raft_consensus.cc:2932] T 9e5fe86b44744df9b2e4d430fdec716f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.175477 22444 catalog_manager.cc:5484] T 9e5fe86b44744df9b2e4d430fdec716f P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 8 to 9, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:50.177604 22979 consensus_queue.cc:237] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:50.179152 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet 9e5fe86b44744df9b2e4d430fdec716f with delete_type TABLET_DATA_TOMBSTONED (TS 725969d490a04fdfb3ba890c3f5096e3 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:42244
W20240502 07:58:50.179279 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:50.179432 22845 raft_consensus.cc:1261] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 07:58:50.179797 29470 tablet_replica.cc:330] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:50.179929 29470 raft_consensus.cc:2229] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:50.180027 24871 raft_consensus.cc:1261] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 07:58:50.180101 29470 raft_consensus.cc:2258] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:50.180421 29334 consensus_queue.cc:1035] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:50.180588 29022 consensus_queue.cc:1035] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 07:58:50.180711 29470 ts_tablet_manager.cc:1875] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
W20240502 07:58:50.181113 22914 consensus_peers.cc:485] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Couldn't send request to peer 7eced220cf1642f3873fc385b6191e69. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e5d5f67e026344998d5fb079f532d076. This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:50.182327 29022 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:50.182972 29470 ts_tablet_manager.cc:1888] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 07:58:50.183027 29470 log.cc:1196] T 9e5fe86b44744df9b2e4d430fdec716f P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/9e5fe86b44744df9b2e4d430fdec716f
I20240502 07:58:50.183681 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 9e5fe86b44744df9b2e4d430fdec716f (table rebalance_test_table_15 [id=4b1b337aafc04dc69a2e37f348a5652a]) successfully deleted
I20240502 07:58:50.184103 22845 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:50.185051 22435 catalog_manager.cc:5484] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:50.186340 24871 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } } }
I20240502 07:58:50.191870 22435 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:50.196223 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 1bb64273325e4586b1776eb64e329231 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=746cab73d7884ee2b46a3feaa4dd9f44]), partition=
I20240502 07:58:50.196281 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 1bb64273325e4586b1776eb64e329231 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=746cab73d7884ee2b46a3feaa4dd9f44]), partition=
I20240502 07:58:50.196437 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1bb64273325e4586b1776eb64e329231. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.196503 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1bb64273325e4586b1776eb64e329231. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.198575 29483 tablet_bootstrap.cc:492] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:50.198577 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 1bb64273325e4586b1776eb64e329231 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=746cab73d7884ee2b46a3feaa4dd9f44]), partition=
I20240502 07:58:50.198755 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1bb64273325e4586b1776eb64e329231. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.198854 29484 tablet_bootstrap.cc:492] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:50.199651 29484 tablet_bootstrap.cc:654] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:50.200201 29483 tablet_bootstrap.cc:654] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:50.200548 29484 tablet_bootstrap.cc:492] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:50.200621 29484 ts_tablet_manager.cc:1367] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:50.200910 29486 tablet_bootstrap.cc:492] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:50.201033 29484 raft_consensus.cc:348] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:50.201094 29484 raft_consensus.cc:374] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.201112 29484 raft_consensus.cc:729] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.201223 29484 consensus_queue.cc:260] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:50.201465 29484 ts_tablet_manager.cc:1398] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:50.201483 29483 tablet_bootstrap.cc:492] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:50.201543 29483 ts_tablet_manager.cc:1367] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:50.201597 29486 tablet_bootstrap.cc:654] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:50.201879 29483 raft_consensus.cc:348] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:50.201953 29483 raft_consensus.cc:374] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.201977 29483 raft_consensus.cc:729] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.202077 29483 consensus_queue.cc:260] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:50.202447 29486 tablet_bootstrap.cc:492] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:50.202498 29486 ts_tablet_manager.cc:1367] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:50.202615 29483 ts_tablet_manager.cc:1398] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:50.202955 29486 raft_consensus.cc:348] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:50.203040 29486 raft_consensus.cc:374] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.203063 29486 raft_consensus.cc:729] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.203143 29486 consensus_queue.cc:260] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:50.203552 29486 ts_tablet_manager.cc:1398] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:50.205729 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_20"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:50.212069 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 7f8b8ce864e24201ba3d318fdce76d66 (DEFAULT_TABLE table=rebalancer_extra_table_20 [id=1e9687ba726b4b2498c75ba480052b2a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:50.212327 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7f8b8ce864e24201ba3d318fdce76d66. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.212342 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 4f3fb3e3b89540a6bd7c6ff623ff4ff7 (DEFAULT_TABLE table=rebalancer_extra_table_20 [id=1e9687ba726b4b2498c75ba480052b2a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:50.212513 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f3fb3e3b89540a6bd7c6ff623ff4ff7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.213251 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 7f8b8ce864e24201ba3d318fdce76d66 (DEFAULT_TABLE table=rebalancer_extra_table_20 [id=1e9687ba726b4b2498c75ba480052b2a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:50.213495 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7f8b8ce864e24201ba3d318fdce76d66. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.214331 29494 tablet_bootstrap.cc:492] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:50.215221 29494 tablet_bootstrap.cc:654] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:50.215806 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 4f3fb3e3b89540a6bd7c6ff623ff4ff7 (DEFAULT_TABLE table=rebalancer_extra_table_20 [id=1e9687ba726b4b2498c75ba480052b2a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:50.216017 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f3fb3e3b89540a6bd7c6ff623ff4ff7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.216850 29496 tablet_bootstrap.cc:492] T 7f8b8ce864e24201ba3d318fdce76d66 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:50.216921 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 266266d3892c47d6826085bdc7fc042e (DEFAULT_TABLE table=rebalancer_extra_table_20 [id=1e9687ba726b4b2498c75ba480052b2a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:50.217155 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 266266d3892c47d6826085bdc7fc042e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.218832 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 266266d3892c47d6826085bdc7fc042e (DEFAULT_TABLE table=rebalancer_extra_table_20 [id=1e9687ba726b4b2498c75ba480052b2a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:50.219048 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 266266d3892c47d6826085bdc7fc042e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.221207 29496 tablet_bootstrap.cc:654] T 7f8b8ce864e24201ba3d318fdce76d66 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:50.221908 29495 tablet_bootstrap.cc:492] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:50.222524 29496 tablet_bootstrap.cc:492] T 7f8b8ce864e24201ba3d318fdce76d66 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:50.222659 29496 ts_tablet_manager.cc:1367] T 7f8b8ce864e24201ba3d318fdce76d66 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:58:50.222764 29483 tablet_bootstrap.cc:492] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:50.222790 29495 tablet_bootstrap.cc:654] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:50.223258 29496 raft_consensus.cc:348] T 7f8b8ce864e24201ba3d318fdce76d66 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:50.223357 29496 raft_consensus.cc:374] T 7f8b8ce864e24201ba3d318fdce76d66 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.223412 29496 raft_consensus.cc:729] T 7f8b8ce864e24201ba3d318fdce76d66 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.223518 29496 consensus_queue.cc:260] T 7f8b8ce864e24201ba3d318fdce76d66 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:50.223634 29483 tablet_bootstrap.cc:654] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:50.223778 29496 ts_tablet_manager.cc:1398] T 7f8b8ce864e24201ba3d318fdce76d66 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:50.225031 29483 tablet_bootstrap.cc:492] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:50.225073 29502 tablet_bootstrap.cc:492] T 266266d3892c47d6826085bdc7fc042e P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:50.225111 29483 ts_tablet_manager.cc:1367] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:50.225446 29483 raft_consensus.cc:348] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:50.225553 29483 raft_consensus.cc:374] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.225586 29483 raft_consensus.cc:729] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.225694 29483 consensus_queue.cc:260] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:50.225906 29502 tablet_bootstrap.cc:654] T 266266d3892c47d6826085bdc7fc042e P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:50.225955 29483 ts_tablet_manager.cc:1398] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:50.227243 29494 tablet_bootstrap.cc:492] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:50.227312 29494 ts_tablet_manager.cc:1367] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.013s user 0.001s sys 0.000s
I20240502 07:58:50.227556 29502 tablet_bootstrap.cc:492] T 266266d3892c47d6826085bdc7fc042e P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:50.227687 29502 ts_tablet_manager.cc:1367] T 266266d3892c47d6826085bdc7fc042e P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:50.227666 29494 raft_consensus.cc:348] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:50.227749 29494 raft_consensus.cc:374] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.227775 29494 raft_consensus.cc:729] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.227869 29494 consensus_queue.cc:260] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:50.228068 29494 ts_tablet_manager.cc:1398] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:50.228190 29494 tablet_bootstrap.cc:492] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:50.229092 29494 tablet_bootstrap.cc:654] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:50.229895 29502 raft_consensus.cc:348] T 266266d3892c47d6826085bdc7fc042e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:50.230005 29494 tablet_bootstrap.cc:492] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:50.230017 29502 raft_consensus.cc:374] T 266266d3892c47d6826085bdc7fc042e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.230055 29502 raft_consensus.cc:729] T 266266d3892c47d6826085bdc7fc042e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.230093 29494 ts_tablet_manager.cc:1367] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:50.230161 29502 consensus_queue.cc:260] T 266266d3892c47d6826085bdc7fc042e P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:50.230532 29502 ts_tablet_manager.cc:1398] T 266266d3892c47d6826085bdc7fc042e P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:50.230648 29494 raft_consensus.cc:348] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:50.230762 29494 raft_consensus.cc:374] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.230813 29494 raft_consensus.cc:729] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.230974 29494 consensus_queue.cc:260] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:50.231235 29494 ts_tablet_manager.cc:1398] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:50.232623 29495 tablet_bootstrap.cc:492] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:50.232740 29495 ts_tablet_manager.cc:1367] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.000s
I20240502 07:58:50.233156 29495 raft_consensus.cc:348] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:50.233242 29495 raft_consensus.cc:374] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.233278 29495 raft_consensus.cc:729] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.233392 29495 consensus_queue.cc:260] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:50.233707 29495 ts_tablet_manager.cc:1398] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:50.240928 25265 raft_consensus.cc:1203] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69 [term 2 LEARNER]: Deduplicated request from leader. Original: 2.7->[2.8-2.8] Dedup: 2.8->[]
W20240502 07:58:50.257367 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:58:50.258263 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:50.276023 22979 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "ca85b40df34d4e1c91a55d7dd69e1494"
dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:46748
I20240502 07:58:50.276156 22979 raft_consensus.cc:595] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:50.289990 27488 raft_consensus.cc:482] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:50.290089 27488 raft_consensus.cc:504] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:50.290445 27488 leader_election.cc:288] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:50.290747 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1bb64273325e4586b1776eb64e329231" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:50.290911 22717 raft_consensus.cc:2454] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:50.290731 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1bb64273325e4586b1776eb64e329231" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:50.291160 24872 raft_consensus.cc:2454] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:50.291388 25068 leader_election.cc:302] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:50.291587 27488 raft_consensus.cc:2781] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:50.291641 27488 raft_consensus.cc:482] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:50.291682 27488 raft_consensus.cc:3037] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:50.291887 29567 raft_consensus.cc:482] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:50.291962 29567 raft_consensus.cc:504] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:50.292508 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f3fb3e3b89540a6bd7c6ff623ff4ff7" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:50.292634 22979 raft_consensus.cc:2454] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:50.292706 27488 raft_consensus.cc:504] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:50.292914 25069 leader_election.cc:302] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 4654231dfe994c1a9d2e37d9c3d75699; no voters:
I20240502 07:58:50.293002 27488 leader_election.cc:288] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:50.293080 27488 raft_consensus.cc:2781] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:50.293078 29567 leader_election.cc:288] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:50.293133 27488 raft_consensus.cc:482] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:50.293186 27488 raft_consensus.cc:3037] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
W20240502 07:58:50.293192 25070 leader_election.cc:334] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:50.293797 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1bb64273325e4586b1776eb64e329231" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:50.293821 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1bb64273325e4586b1776eb64e329231" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:50.293906 24872 raft_consensus.cc:3037] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:50.293923 22717 raft_consensus.cc:3037] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:50.293926 27488 raft_consensus.cc:504] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:50.294504 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f3fb3e3b89540a6bd7c6ff623ff4ff7" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:50.294593 22979 raft_consensus.cc:3037] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:50.294874 22717 raft_consensus.cc:2454] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:50.294893 24872 raft_consensus.cc:2454] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:50.295131 25068 leader_election.cc:302] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:50.295544 29568 raft_consensus.cc:2781] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:50.295612 22979 raft_consensus.cc:2454] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
W20240502 07:58:50.295831 25070 leader_election.cc:334] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:50.295899 25069 leader_election.cc:302] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 4654231dfe994c1a9d2e37d9c3d75699; no voters: 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:50.296152 27488 leader_election.cc:288] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:50.296205 29568 raft_consensus.cc:686] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:58:50.296300 29569 raft_consensus.cc:2781] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:50.296594 29568 consensus_queue.cc:237] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:50.297434 29569 raft_consensus.cc:686] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:58:50.297636 29569 consensus_queue.cc:237] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:50.298612 22440 catalog_manager.cc:5484] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:50.302310 22435 catalog_manager.cc:5484] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:50.333247 22979 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "ca85b40df34d4e1c91a55d7dd69e1494"
dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:46796
I20240502 07:58:50.333357 22979 raft_consensus.cc:595] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Received request to transfer leadership
I20240502 07:58:50.340243 27495 raft_consensus.cc:482] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:50.340320 27495 raft_consensus.cc:504] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:50.340570 27495 leader_election.cc:288] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:50.340854 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "266266d3892c47d6826085bdc7fc042e" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:50.340967 22845 raft_consensus.cc:2454] T 266266d3892c47d6826085bdc7fc042e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:50.341184 25202 leader_election.cc:302] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:50.341315 27495 raft_consensus.cc:2781] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:50.341368 27495 raft_consensus.cc:482] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:50.341413 27495 raft_consensus.cc:3037] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:50.342103 27495 raft_consensus.cc:504] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:50.342335 27495 leader_election.cc:288] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
W20240502 07:58:50.342418 25201 leader_election.cc:334] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:50.342553 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "266266d3892c47d6826085bdc7fc042e" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:50.342634 22845 raft_consensus.cc:3037] T 266266d3892c47d6826085bdc7fc042e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
W20240502 07:58:50.343348 25201 leader_election.cc:334] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:50.343585 22845 raft_consensus.cc:2454] T 266266d3892c47d6826085bdc7fc042e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:50.343827 25202 leader_election.cc:302] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 7eced220cf1642f3873fc385b6191e69; no voters: 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:50.343971 27495 raft_consensus.cc:2781] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:50.344070 27495 raft_consensus.cc:686] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:50.344201 27495 consensus_queue.cc:237] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } }
I20240502 07:58:50.345916 22440 catalog_manager.cc:5484] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:50.406737 27495 raft_consensus.cc:482] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:50.406873 27495 raft_consensus.cc:504] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:50.406922 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:50.407284 27495 leader_election.cc:288] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:50.407980 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7f8b8ce864e24201ba3d318fdce76d66" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:50.408114 22586 raft_consensus.cc:2454] T 7f8b8ce864e24201ba3d318fdce76d66 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
W20240502 07:58:50.408360 25201 leader_election.cc:334] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:50.408445 25202 leader_election.cc:302] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7eced220cf1642f3873fc385b6191e69, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters: 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:50.408582 27495 raft_consensus.cc:2781] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:50.408622 27495 raft_consensus.cc:482] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:50.408704 27495 raft_consensus.cc:3037] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:50.409773 27495 raft_consensus.cc:504] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:50.410362 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7f8b8ce864e24201ba3d318fdce76d66" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:50.410454 22586 raft_consensus.cc:3037] T 7f8b8ce864e24201ba3d318fdce76d66 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:50.411091 24872 raft_consensus.cc:1261] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:50.411345 22586 raft_consensus.cc:2454] T 7f8b8ce864e24201ba3d318fdce76d66 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
W20240502 07:58:50.411387 25201 leader_election.cc:334] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:50.411506 29571 consensus_queue.cc:1035] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:50.411609 25202 leader_election.cc:302] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7eced220cf1642f3873fc385b6191e69, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters: 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:50.411741 27495 leader_election.cc:288] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:50.411831 27495 raft_consensus.cc:2781] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:50.411821 22716 raft_consensus.cc:1261] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:50.412122 27495 raft_consensus.cc:686] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:50.412262 27495 consensus_queue.cc:237] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:50.412338 29571 consensus_queue.cc:1035] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:50.415256 22440 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet 1bb64273325e4586b1776eb64e329231 (table rebalancer_dynamic_table [id=746cab73d7884ee2b46a3feaa4dd9f44]). Expected version 1 got 0
I20240502 07:58:50.416019 22442 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet 1bb64273325e4586b1776eb64e329231 (table rebalancer_dynamic_table [id=746cab73d7884ee2b46a3feaa4dd9f44]). Expected version 1 got 0
I20240502 07:58:50.416153 22435 catalog_manager.cc:5484] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:50.417819 29208 tablet.cc:1722] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:50.418109 29208 tablet.cc:1620] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:50.418372 22444 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 1bb64273325e4586b1776eb64e329231 (table rebalancer_dynamic_table [id=746cab73d7884ee2b46a3feaa4dd9f44]). Expected version 1 got 0
I20240502 07:58:50.420526 29617 mvcc.cc:205] Tried to move back new op lower bound from 7023152047759937536 to 7023152047296356352. Current Snapshot: MvccSnapshot[applied={T|T < 7023152047759937536}]
I20240502 07:58:50.421299 22436 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 1bb64273325e4586b1776eb64e329231 (table rebalancer_dynamic_table [id=746cab73d7884ee2b46a3feaa4dd9f44]). Expected version 1 got 0
I20240502 07:58:50.421502 22435 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=746cab73d7884ee2b46a3feaa4dd9f44] alter complete (version 1)
I20240502 07:58:50.422231 22444 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet 1bb64273325e4586b1776eb64e329231 (table rebalancer_dynamic_table [id=746cab73d7884ee2b46a3feaa4dd9f44]). Expected version 1 got 0
I20240502 07:58:50.423269 29620 tablet.cc:1722] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:50.423533 29620 tablet.cc:1620] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:50.436800 29621 tablet.cc:1722] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:50.437079 29621 tablet.cc:1620] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:58:50.513144 25134 raft_consensus.cc:1203] T 539cf5d4596f481cb03ab4d13a19198c P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 07:58:50.523545 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:50.528492 29620 tablet.cc:1722] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:50.528888 29620 tablet.cc:1620] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:50.528900 29208 tablet.cc:1722] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:50.529165 29208 tablet.cc:1620] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:50.529382 29621 tablet.cc:1722] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:50.529683 29621 tablet.cc:1620] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:58:50.530473 29334 raft_consensus.cc:979] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f: : Instructing follower 7db4178eed174225bfb24f5f9445ce88 to start an election
I20240502 07:58:50.530586 29334 raft_consensus.cc:1067] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Signalling peer 7db4178eed174225bfb24f5f9445ce88 to start an election
I20240502 07:58:50.531067 22845 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "ca85b40df34d4e1c91a55d7dd69e1494"
dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
from {username='slave'} at 127.16.85.132:41687
I20240502 07:58:50.531216 22845 raft_consensus.cc:482] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 07:58:50.531275 22845 raft_consensus.cc:3037] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:50.532239 22845 raft_consensus.cc:504] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:50.532642 22845 leader_election.cc:288] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 2 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:50.533159 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca85b40df34d4e1c91a55d7dd69e1494" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:50.533284 24872 raft_consensus.cc:3037] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:50.533726 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca85b40df34d4e1c91a55d7dd69e1494" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:50.533830 22979 raft_consensus.cc:3032] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Stepping down as leader of term 1
I20240502 07:58:50.533869 22979 raft_consensus.cc:729] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:50.534004 22979 consensus_queue.cc:260] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:50.534201 22979 raft_consensus.cc:3037] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:50.534219 24872 raft_consensus.cc:2454] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 2.
I20240502 07:58:50.534585 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=746cab73d7884ee2b46a3feaa4dd9f44] alter complete (version 2)
I20240502 07:58:50.535213 22979 raft_consensus.cc:2454] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 2.
I20240502 07:58:50.535794 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca85b40df34d4e1c91a55d7dd69e1494" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:50.535924 25265 raft_consensus.cc:3037] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:50.536682 25265 raft_consensus.cc:2454] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 2.
I20240502 07:58:50.536901 22782 leader_election.cc:302] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:50.537057 29566 raft_consensus.cc:2781] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:50.537155 29566 raft_consensus.cc:686] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 2 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:50.537298 29566 consensus_queue.cc:237] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:50.539002 22440 catalog_manager.cc:5484] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 1 to 2, leader changed from 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 2 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:50.609076 29672 ts_tablet_manager.cc:902] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:50.609494 29672 tablet_copy_client.cc:308] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:50.609895 22999 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet 6af67c65672f45a68e960936351e223f from peer 7db4178eed174225bfb24f5f9445ce88 ({username='slave'} at 127.16.85.131:49181)
I20240502 07:58:50.609977 22999 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet 6af67c65672f45a68e960936351e223f from peer 7db4178eed174225bfb24f5f9445ce88 at {username='slave'} at 127.16.85.131:49181: session id = 7db4178eed174225bfb24f5f9445ce88-6af67c65672f45a68e960936351e223f
I20240502 07:58:50.610615 22999 tablet_copy_source_session.cc:205] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:50.611367 29672 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6af67c65672f45a68e960936351e223f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.613222 29672 tablet_copy_client.cc:783] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:50.613358 29672 tablet_copy_client.cc:647] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:50.614272 29672 tablet_copy_client.cc:523] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:50.615197 29672 tablet_bootstrap.cc:492] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:50.617748 29672 tablet_bootstrap.cc:492] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:50.617938 29672 tablet_bootstrap.cc:492] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88: Bootstrap complete.
I20240502 07:58:50.618065 29672 ts_tablet_manager.cc:1367] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.003s sys 0.000s
I20240502 07:58:50.618371 29672 raft_consensus.cc:348] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 07:58:50.618482 29672 raft_consensus.cc:729] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: LEARNER
I20240502 07:58:50.618575 29672 consensus_queue.cc:260] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 07:58:50.618824 29672 ts_tablet_manager.cc:1398] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:50.619192 22999 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 7db4178eed174225bfb24f5f9445ce88-6af67c65672f45a68e960936351e223f received from {username='slave'} at 127.16.85.131:49181
I20240502 07:58:50.619261 22999 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 7db4178eed174225bfb24f5f9445ce88-6af67c65672f45a68e960936351e223f on tablet 6af67c65672f45a68e960936351e223f with peer 7db4178eed174225bfb24f5f9445ce88
I20240502 07:58:50.633699 29334 raft_consensus.cc:1050] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 7eced220cf1642f3873fc385b6191e69 to VOTER
I20240502 07:58:50.634058 29334 consensus_queue.cc:237] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:50.635231 25265 raft_consensus.cc:1261] T a8dcc36740be4ca3ac1d43591f7549bd P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:50.635229 22716 raft_consensus.cc:1261] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:50.635588 27333 consensus_queue.cc:1035] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:50.635828 29334 consensus_queue.cc:1035] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240502 07:58:50.636008 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:50.636236 29679 ts_tablet_manager.cc:902] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:50.636375 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:50.636689 29679 tablet_copy_client.cc:308] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 07:58:50.637235 22999 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet e5d5f67e026344998d5fb079f532d076 from peer 7eced220cf1642f3873fc385b6191e69 ({username='slave'} at 127.16.85.136:40185)
I20240502 07:58:50.637312 22999 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet e5d5f67e026344998d5fb079f532d076 from peer 7eced220cf1642f3873fc385b6191e69 at {username='slave'} at 127.16.85.136:40185: session id = 7eced220cf1642f3873fc385b6191e69-e5d5f67e026344998d5fb079f532d076
I20240502 07:58:50.638082 22999 tablet_copy_source_session.cc:205] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 07:58:50.639139 29334 raft_consensus.cc:2932] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.640370 29679 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e5d5f67e026344998d5fb079f532d076. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.641294 22717 raft_consensus.cc:2932] T a8dcc36740be4ca3ac1d43591f7549bd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.641793 29620 tablet.cc:1722] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:50.642657 29208 tablet.cc:1722] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:50.642902 29679 tablet_copy_client.cc:783] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 0 data blocks...
I20240502 07:58:50.643028 29679 tablet_copy_client.cc:647] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Starting download of 1 WAL segments...
I20240502 07:58:50.643173 25265 raft_consensus.cc:2932] T a8dcc36740be4ca3ac1d43591f7549bd P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.644136 29679 tablet_copy_client.cc:523] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 07:58:50.645610 29679 tablet_bootstrap.cc:492] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:50.646545 22442 catalog_manager.cc:5484] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:50.647948 29621 tablet.cc:1722] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:50.649410 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=746cab73d7884ee2b46a3feaa4dd9f44] alter complete (version 3)
W20240502 07:58:50.660450 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:50.663664 29571 raft_consensus.cc:482] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 74f9aa91c2c448f980b512fe2e8d672e)
I20240502 07:58:50.663771 29571 raft_consensus.cc:504] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } }
I20240502 07:58:50.664176 29571 leader_election.cc:288] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:50.664779 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28832d9e18ea41b194cf33d8c7971e23" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
W20240502 07:58:50.665338 25070 leader_election.cc:334] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:50.665405 25070 leader_election.cc:302] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699; no voters: 74f9aa91c2c448f980b512fe2e8d672e, 7db4178eed174225bfb24f5f9445ce88
I20240502 07:58:50.665524 29571 raft_consensus.cc:2726] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 07:58:50.666975 29679 tablet_bootstrap.cc:492] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 07:58:50.667213 29679 tablet_bootstrap.cc:492] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69: Bootstrap complete.
I20240502 07:58:50.667351 29679 ts_tablet_manager.cc:1367] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.022s user 0.015s sys 0.003s
I20240502 07:58:50.667672 29679 raft_consensus.cc:348] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:50.667760 29679 raft_consensus.cc:729] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: LEARNER
I20240502 07:58:50.667829 29679 consensus_queue.cc:260] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: NON_VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: true } }
I20240502 07:58:50.668074 29679 ts_tablet_manager.cc:1398] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:50.668481 22999 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 7eced220cf1642f3873fc385b6191e69-e5d5f67e026344998d5fb079f532d076 received from {username='slave'} at 127.16.85.136:40185
I20240502 07:58:50.668561 22999 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 7eced220cf1642f3873fc385b6191e69-e5d5f67e026344998d5fb079f532d076 on tablet e5d5f67e026344998d5fb079f532d076 with peer 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:50.688571 26995 raft_consensus.cc:1050] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3: attempting to promote NON_VOTER 7eced220cf1642f3873fc385b6191e69 to VOTER
I20240502 07:58:50.688935 26995 consensus_queue.cc:237] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 5, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:50.690107 24871 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 2 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 07:58:50.690181 22845 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 07:58:50.690533 27303 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:50.690660 26995 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:50.690719 25265 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69 [term 2 LEARNER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 07:58:50.691051 29689 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:50.692771 27303 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 2 LEADER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.693113 25265 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.693425 22845 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.696059 24871 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.696395 22436 catalog_manager.cc:5484] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 8 to 9, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New cstate: current_term: 2 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:50.752761 22440 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:50.752928 22440 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:50.754932 22440 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 1bb64273325e4586b1776eb64e329231
I20240502 07:58:50.755460 25115 tablet_service.cc:1514] Processing DeleteTablet for tablet 1bb64273325e4586b1776eb64e329231 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:50 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:50.755726 24850 tablet_service.cc:1514] Processing DeleteTablet for tablet 1bb64273325e4586b1776eb64e329231 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:50 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:50.755745 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet 1bb64273325e4586b1776eb64e329231 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:50 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:50.755825 29693 tablet_replica.cc:330] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:50.755960 29693 raft_consensus.cc:2229] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:50.756016 29695 tablet_replica.cc:330] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:50.756140 29695 raft_consensus.cc:2229] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:50.756160 29693 raft_consensus.cc:2258] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:50.756325 29695 raft_consensus.cc:2258] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:50.756383 29694 tablet_replica.cc:330] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:50.756516 29694 raft_consensus.cc:2229] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:50.756649 29694 raft_consensus.cc:2258] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:50.756937 29693 ts_tablet_manager.cc:1875] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:50.757179 29695 ts_tablet_manager.cc:1875] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:50.757232 29694 ts_tablet_manager.cc:1875] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:50.759241 29693 ts_tablet_manager.cc:1888] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:50.759357 29693 log.cc:1196] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/1bb64273325e4586b1776eb64e329231
I20240502 07:58:50.759409 29694 ts_tablet_manager.cc:1888] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:50.759518 29694 log.cc:1196] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/1bb64273325e4586b1776eb64e329231
I20240502 07:58:50.759552 29695 ts_tablet_manager.cc:1888] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:50.759639 29695 log.cc:1196] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/1bb64273325e4586b1776eb64e329231
I20240502 07:58:50.759670 29693 ts_tablet_manager.cc:1909] T 1bb64273325e4586b1776eb64e329231 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:50.759785 29694 ts_tablet_manager.cc:1909] T 1bb64273325e4586b1776eb64e329231 P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:50.759943 29695 ts_tablet_manager.cc:1909] T 1bb64273325e4586b1776eb64e329231 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:50.760313 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet 1bb64273325e4586b1776eb64e329231 (table rebalancer_dynamic_table_new_name [id=746cab73d7884ee2b46a3feaa4dd9f44]) successfully deleted
I20240502 07:58:50.760336 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet 1bb64273325e4586b1776eb64e329231 (table rebalancer_dynamic_table_new_name [id=746cab73d7884ee2b46a3feaa4dd9f44]) successfully deleted
I20240502 07:58:50.760463 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 1bb64273325e4586b1776eb64e329231 (table rebalancer_dynamic_table_new_name [id=746cab73d7884ee2b46a3feaa4dd9f44]) successfully deleted
I20240502 07:58:50.792529 28205 consensus_queue.cc:1035] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:58:50.796205 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:50.818557 28408 raft_consensus.cc:482] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 74f9aa91c2c448f980b512fe2e8d672e)
I20240502 07:58:50.818691 28408 raft_consensus.cc:504] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } }
I20240502 07:58:50.819123 28408 leader_election.cc:288] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:50.819437 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28832d9e18ea41b194cf33d8c7971e23" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:50.819578 25134 raft_consensus.cc:2454] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:50.819885 22784 leader_election.cc:302] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7db4178eed174225bfb24f5f9445ce88; no voters:
W20240502 07:58:50.820013 22783 leader_election.cc:334] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:50.820453 28408 raft_consensus.cc:2781] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20240502 07:58:50.820521 28408 raft_consensus.cc:482] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 74f9aa91c2c448f980b512fe2e8d672e)
I20240502 07:58:50.820580 28408 raft_consensus.cc:3037] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:50.821591 28408 raft_consensus.cc:504] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Starting leader election with config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } }
I20240502 07:58:50.821947 28408 leader_election.cc:288] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 2 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:50.822320 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28832d9e18ea41b194cf33d8c7971e23" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:50.822405 25134 raft_consensus.cc:3037] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:50.823302 25134 raft_consensus.cc:2454] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 2.
I20240502 07:58:50.823570 22784 leader_election.cc:302] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:50.823753 28408 raft_consensus.cc:2781] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:50.823832 28408 raft_consensus.cc:686] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [term 2 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:50.823971 28408 consensus_queue.cc:237] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } }
W20240502 07:58:50.824331 22783 leader_election.cc:334] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:50.825800 22435 catalog_manager.cc:5484] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 1 to 2, leader changed from 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 2 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:50.826504 29571 consensus_queue.cc:1035] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:58:50.830147 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:50.843680 22717 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "9d2283e5d28347519b38e876d840608e"
dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:53964
I20240502 07:58:50.843827 22717 raft_consensus.cc:595] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 2 LEADER]: Received request to transfer leadership
I20240502 07:58:50.856420 22435 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:50.860834 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 41284d865ebb4cd18f2ce08f66343262 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=a295d5f8cc2c4b429ea679bc6e4a49cc]), partition=
I20240502 07:58:50.861135 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 41284d865ebb4cd18f2ce08f66343262. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.861116 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 41284d865ebb4cd18f2ce08f66343262 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=a295d5f8cc2c4b429ea679bc6e4a49cc]), partition=
I20240502 07:58:50.861348 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 41284d865ebb4cd18f2ce08f66343262. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.862982 29759 tablet_bootstrap.cc:492] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:50.863876 29760 tablet_bootstrap.cc:492] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:50.864148 29759 tablet_bootstrap.cc:654] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
W20240502 07:58:50.864465 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:50.864759 29760 tablet_bootstrap.cc:654] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:50.865207 29759 tablet_bootstrap.cc:492] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:50.865298 29759 ts_tablet_manager.cc:1367] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:50.865718 29760 tablet_bootstrap.cc:492] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:50.865705 29759 raft_consensus.cc:348] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:50.865779 29759 raft_consensus.cc:374] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.865796 29760 ts_tablet_manager.cc:1367] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:50.865818 29759 raft_consensus.cc:729] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.865933 29759 consensus_queue.cc:260] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:50.866196 29759 ts_tablet_manager.cc:1398] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:50.866240 29760 raft_consensus.cc:348] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:50.866366 29760 raft_consensus.cc:374] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.866418 29760 raft_consensus.cc:729] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.866544 29760 consensus_queue.cc:260] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:50.866974 29760 ts_tablet_manager.cc:1398] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:50.893330 29570 raft_consensus.cc:482] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 74f9aa91c2c448f980b512fe2e8d672e)
I20240502 07:58:50.893445 29570 raft_consensus.cc:504] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 07:58:50.893842 29570 leader_election.cc:288] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:50.894122 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "90acd69ebaba4fb2a67e16172e8d7aa5" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 2 candidate_status { last_received { term: 1 index: 2 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
W20240502 07:58:50.894781 25070 leader_election.cc:334] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:50.894869 25070 leader_election.cc:302] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699; no voters: 6d02017318a2461796d431962fe6a537, 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:50.894987 29570 raft_consensus.cc:2726] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 07:58:50.923692 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_22"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:50.928947 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 7ecf0142784a4dacbb61f15215dbe24c (DEFAULT_TABLE table=rebalancer_extra_table_22 [id=4759071c6b624e36896d3b2a0fca2b6b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:50.929232 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ecf0142784a4dacbb61f15215dbe24c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.929598 24850 tablet_service.cc:1467] Processing CreateTablet for tablet fc05622956224d2ba1d829aaf1ddea3e (DEFAULT_TABLE table=rebalancer_extra_table_22 [id=4759071c6b624e36896d3b2a0fca2b6b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:50.929733 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 7ecf0142784a4dacbb61f15215dbe24c (DEFAULT_TABLE table=rebalancer_extra_table_22 [id=4759071c6b624e36896d3b2a0fca2b6b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:50.929803 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc05622956224d2ba1d829aaf1ddea3e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.930019 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ecf0142784a4dacbb61f15215dbe24c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.931341 29760 tablet_bootstrap.cc:492] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:50.931628 29759 tablet_bootstrap.cc:492] T fc05622956224d2ba1d829aaf1ddea3e P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:50.932188 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 68cfb914f8c140728ba025b37d7da7f5 (DEFAULT_TABLE table=rebalancer_extra_table_22 [id=4759071c6b624e36896d3b2a0fca2b6b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:50.932310 29760 tablet_bootstrap.cc:654] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:50.932405 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 68cfb914f8c140728ba025b37d7da7f5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.932608 29759 tablet_bootstrap.cc:654] T fc05622956224d2ba1d829aaf1ddea3e P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:50.933389 28205 consensus_queue.cc:1035] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:50.933939 29763 tablet_bootstrap.cc:492] T 7ecf0142784a4dacbb61f15215dbe24c P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:50.934144 29759 tablet_bootstrap.cc:492] T fc05622956224d2ba1d829aaf1ddea3e P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:50.934274 29759 ts_tablet_manager.cc:1367] T fc05622956224d2ba1d829aaf1ddea3e P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.001s
I20240502 07:58:50.935238 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 68cfb914f8c140728ba025b37d7da7f5 (DEFAULT_TABLE table=rebalancer_extra_table_22 [id=4759071c6b624e36896d3b2a0fca2b6b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:50.935478 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 68cfb914f8c140728ba025b37d7da7f5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.937144 25235 tablet_service.cc:1467] Processing CreateTablet for tablet fc05622956224d2ba1d829aaf1ddea3e (DEFAULT_TABLE table=rebalancer_extra_table_22 [id=4759071c6b624e36896d3b2a0fca2b6b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:50.937356 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc05622956224d2ba1d829aaf1ddea3e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:50.937433 29760 tablet_bootstrap.cc:492] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:50.937505 29760 ts_tablet_manager.cc:1367] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.002s
I20240502 07:58:50.937928 29760 raft_consensus.cc:348] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:50.938017 29760 raft_consensus.cc:374] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.938050 29760 raft_consensus.cc:729] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.938171 29760 consensus_queue.cc:260] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:50.938444 29760 ts_tablet_manager.cc:1398] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:50.938576 29760 tablet_bootstrap.cc:492] T 68cfb914f8c140728ba025b37d7da7f5 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:50.938941 24872 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537 [term 2 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 07:58:50.939349 28408 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:50.939467 29760 tablet_bootstrap.cc:654] T 68cfb914f8c140728ba025b37d7da7f5 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:50.939924 29763 tablet_bootstrap.cc:654] T 7ecf0142784a4dacbb61f15215dbe24c P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:50.940176 29759 raft_consensus.cc:348] T fc05622956224d2ba1d829aaf1ddea3e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:50.940284 29759 raft_consensus.cc:374] T fc05622956224d2ba1d829aaf1ddea3e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.940315 29759 raft_consensus.cc:729] T fc05622956224d2ba1d829aaf1ddea3e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.940414 29759 consensus_queue.cc:260] T fc05622956224d2ba1d829aaf1ddea3e P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:50.940708 29759 ts_tablet_manager.cc:1398] T fc05622956224d2ba1d829aaf1ddea3e P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:58:50.941211 29759 tablet_bootstrap.cc:492] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:50.941833 29760 tablet_bootstrap.cc:492] T 68cfb914f8c140728ba025b37d7da7f5 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:50.941936 29760 ts_tablet_manager.cc:1367] T 68cfb914f8c140728ba025b37d7da7f5 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240502 07:58:50.942339 29759 tablet_bootstrap.cc:654] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:50.942343 29760 raft_consensus.cc:348] T 68cfb914f8c140728ba025b37d7da7f5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:50.942421 29760 raft_consensus.cc:374] T 68cfb914f8c140728ba025b37d7da7f5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.942458 29760 raft_consensus.cc:729] T 68cfb914f8c140728ba025b37d7da7f5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.942579 29760 consensus_queue.cc:260] T 68cfb914f8c140728ba025b37d7da7f5 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:50.942873 29760 ts_tablet_manager.cc:1398] T 68cfb914f8c140728ba025b37d7da7f5 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:50.942996 29760 tablet_bootstrap.cc:492] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:50.943766 29760 tablet_bootstrap.cc:654] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:50.944002 29763 tablet_bootstrap.cc:492] T 7ecf0142784a4dacbb61f15215dbe24c P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:50.944083 29763 ts_tablet_manager.cc:1367] T 7ecf0142784a4dacbb61f15215dbe24c P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.000s
I20240502 07:58:50.944685 29763 raft_consensus.cc:348] T 7ecf0142784a4dacbb61f15215dbe24c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:50.944797 29763 raft_consensus.cc:374] T 7ecf0142784a4dacbb61f15215dbe24c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.944855 29763 raft_consensus.cc:729] T 7ecf0142784a4dacbb61f15215dbe24c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.944980 29763 consensus_queue.cc:260] T 7ecf0142784a4dacbb61f15215dbe24c P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:50.945286 29763 ts_tablet_manager.cc:1398] T 7ecf0142784a4dacbb61f15215dbe24c P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:50.945806 29760 tablet_bootstrap.cc:492] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:50.946028 29760 ts_tablet_manager.cc:1367] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240502 07:58:50.946478 29760 raft_consensus.cc:348] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:50.946636 29760 raft_consensus.cc:374] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.946686 29760 raft_consensus.cc:729] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.946894 29760 consensus_queue.cc:260] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:50.947315 29760 ts_tablet_manager.cc:1398] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:50.948905 29759 tablet_bootstrap.cc:492] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:50.949014 29759 ts_tablet_manager.cc:1367] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.000s
I20240502 07:58:50.949453 29759 raft_consensus.cc:348] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:50.949534 29759 raft_consensus.cc:374] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:50.949556 29759 raft_consensus.cc:729] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:50.949666 29759 consensus_queue.cc:260] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:50.949911 29759 ts_tablet_manager.cc:1398] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
W20240502 07:58:50.960844 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): CreateTablet RPC failed for tablet 7ecf0142784a4dacbb61f15215dbe24c: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:50.967787 28823 raft_consensus.cc:482] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 74f9aa91c2c448f980b512fe2e8d672e)
I20240502 07:58:50.967869 28823 raft_consensus.cc:504] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 07:58:50.968148 28823 leader_election.cc:288] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:50.968492 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "90acd69ebaba4fb2a67e16172e8d7aa5" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 2 candidate_status { last_received { term: 1 index: 2 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:50.968632 25134 raft_consensus.cc:2454] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:50.968907 24808 leader_election.cc:302] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
W20240502 07:58:50.968991 24807 leader_election.cc:334] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:50.969071 28823 raft_consensus.cc:2781] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20240502 07:58:50.969123 28823 raft_consensus.cc:482] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 74f9aa91c2c448f980b512fe2e8d672e)
I20240502 07:58:50.969164 28823 raft_consensus.cc:3037] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:50.970113 28823 raft_consensus.cc:504] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 2 FOLLOWER]: Starting leader election with config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 07:58:50.970455 28823 leader_election.cc:288] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 2 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:50.970757 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "90acd69ebaba4fb2a67e16172e8d7aa5" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 2 candidate_status { last_received { term: 1 index: 2 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:50.970868 25134 raft_consensus.cc:3037] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Advancing to term 2
W20240502 07:58:50.971319 24807 leader_election.cc:334] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:50.971907 25134 raft_consensus.cc:2454] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 2.
I20240502 07:58:50.972151 24808 leader_election.cc:302] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters: 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:50.972393 28823 raft_consensus.cc:2781] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:50.972476 28823 raft_consensus.cc:686] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 2 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:58:50.972627 28823 consensus_queue.cc:237] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 07:58:50.974421 22440 catalog_manager.cc:5484] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 1 to 2, leader changed from 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 2 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:50.977242 25265 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [term 2 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 07:58:50.977578 28408 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:50.979975 22979 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 2 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 07:58:50.981259 29177 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:50.981829 22845 consensus_queue.cc:237] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } }
I20240502 07:58:50.983031 24872 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537 [term 2 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 07:58:50.983139 25265 raft_consensus.cc:1261] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [term 2 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 07:58:50.983556 29773 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 07:58:50.983973 28408 consensus_queue.cc:1035] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 07:58:50.985514 28408 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:50.987267 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ca85b40df34d4e1c91a55d7dd69e1494 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:50.987460 24872 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 6d02017318a2461796d431962fe6a537 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:50.987740 25265 raft_consensus.cc:2932] T ca85b40df34d4e1c91a55d7dd69e1494 P 7eced220cf1642f3873fc385b6191e69 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } }
I20240502 07:58:50.988277 22442 catalog_manager.cc:5484] T ca85b40df34d4e1c91a55d7dd69e1494 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 8 to 10, VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) evicted. New cstate: current_term: 2 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:50.990350 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet ca85b40df34d4e1c91a55d7dd69e1494 with delete_type TABLET_DATA_TOMBSTONED (TS 02a92e1c084b4308b9e855a4cc2c295f not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:50.990633 29774 tablet_replica.cc:330] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:50.990783 29774 raft_consensus.cc:2229] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:50.991073 29774 raft_consensus.cc:2258] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:50.991657 29774 ts_tablet_manager.cc:1875] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:50.993451 29774 ts_tablet_manager.cc:1888] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 07:58:50.993515 29774 log.cc:1196] T ca85b40df34d4e1c91a55d7dd69e1494 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/ca85b40df34d4e1c91a55d7dd69e1494
I20240502 07:58:50.994055 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet ca85b40df34d4e1c91a55d7dd69e1494 (table rebalance_test_table_3 [id=8f253ed078dd4e948cfdd6ff6c19f874]) successfully deleted
I20240502 07:58:51.005652 27808 raft_consensus.cc:482] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:51.005729 27808 raft_consensus.cc:504] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:51.006031 27808 leader_election.cc:288] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:51.006347 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ecf0142784a4dacbb61f15215dbe24c" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:51.006458 22717 raft_consensus.cc:2454] T 7ecf0142784a4dacbb61f15215dbe24c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:51.006677 25199 leader_election.cc:302] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:51.006829 27808 raft_consensus.cc:2781] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:51.006883 27808 raft_consensus.cc:482] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:51.006927 27808 raft_consensus.cc:3037] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
W20240502 07:58:51.006973 25201 leader_election.cc:334] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:51.007589 27808 raft_consensus.cc:504] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:51.007812 27808 leader_election.cc:288] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:51.008155 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ecf0142784a4dacbb61f15215dbe24c" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:51.008270 22717 raft_consensus.cc:3037] T 7ecf0142784a4dacbb61f15215dbe24c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
W20240502 07:58:51.008636 25201 leader_election.cc:334] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:51.009197 22717 raft_consensus.cc:2454] T 7ecf0142784a4dacbb61f15215dbe24c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:51.009444 25199 leader_election.cc:302] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7eced220cf1642f3873fc385b6191e69; no voters: 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:51.009622 27808 raft_consensus.cc:2781] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:51.009721 27808 raft_consensus.cc:686] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:51.009835 27808 consensus_queue.cc:237] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:51.011606 22435 catalog_manager.cc:5484] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:51.075541 27338 raft_consensus.cc:482] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:51.075663 27338 raft_consensus.cc:504] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:51.076037 27338 leader_election.cc:288] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:51.076548 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "68cfb914f8c140728ba025b37d7da7f5" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:51.076716 25265 raft_consensus.cc:2454] T 68cfb914f8c140728ba025b37d7da7f5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
W20240502 07:58:51.076956 24807 leader_election.cc:334] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:51.077117 24807 leader_election.cc:302] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters: 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:51.077246 27338 raft_consensus.cc:2781] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:51.077296 27338 raft_consensus.cc:482] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:51.077330 27338 raft_consensus.cc:3037] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:51.078207 27338 raft_consensus.cc:504] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:51.078492 27338 leader_election.cc:288] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:51.079146 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "68cfb914f8c140728ba025b37d7da7f5" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:51.079248 25265 raft_consensus.cc:3037] T 68cfb914f8c140728ba025b37d7da7f5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
W20240502 07:58:51.079710 24807 leader_election.cc:334] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:51.080209 25265 raft_consensus.cc:2454] T 68cfb914f8c140728ba025b37d7da7f5 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:51.080443 24807 leader_election.cc:302] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters: 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:51.080593 27338 raft_consensus.cc:2781] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:51.080754 27338 raft_consensus.cc:686] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:58:51.080912 27338 consensus_queue.cc:237] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:51.082988 22435 catalog_manager.cc:5484] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:51.097281 29689 raft_consensus.cc:1050] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3: attempting to promote NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 to VOTER
I20240502 07:58:51.097715 29689 consensus_queue.cc:237] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 07:58:51.099079 25134 raft_consensus.cc:1261] T 539cf5d4596f481cb03ab4d13a19198c P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEARNER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:51.099289 22845 raft_consensus.cc:1261] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240502 07:58:51.099716 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:51.099864 27303 consensus_queue.cc:1035] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:51.099989 26995 consensus_queue.cc:1035] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:51.101974 27303 raft_consensus.cc:2932] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:51.102411 22845 raft_consensus.cc:2932] T 539cf5d4596f481cb03ab4d13a19198c P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:51.104326 25134 raft_consensus.cc:2932] T 539cf5d4596f481cb03ab4d13a19198c P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 07:58:51.105095 27808 raft_consensus.cc:482] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:51.105178 27808 raft_consensus.cc:504] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:51.105159 22440 catalog_manager.cc:5484] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 7 to 8, 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:51.105494 27808 leader_election.cc:288] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:51.105834 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc05622956224d2ba1d829aaf1ddea3e" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:51.105980 24872 raft_consensus.cc:2454] T fc05622956224d2ba1d829aaf1ddea3e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:51.106279 25199 leader_election.cc:302] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
W20240502 07:58:51.106699 25201 leader_election.cc:334] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:51.107132 27808 raft_consensus.cc:2781] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:51.107192 27808 raft_consensus.cc:482] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:51.107229 27808 raft_consensus.cc:3037] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:51.108165 27808 raft_consensus.cc:504] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:51.108490 27808 leader_election.cc:288] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:51.108830 22717 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "9d2283e5d28347519b38e876d840608e"
dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:54020
I20240502 07:58:51.108925 22717 raft_consensus.cc:595] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 2 LEADER]: Received request to transfer leadership
I20240502 07:58:51.109409 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc05622956224d2ba1d829aaf1ddea3e" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:51.109522 24872 raft_consensus.cc:3037] T fc05622956224d2ba1d829aaf1ddea3e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
W20240502 07:58:51.110419 25201 leader_election.cc:334] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:51.110558 24872 raft_consensus.cc:2454] T fc05622956224d2ba1d829aaf1ddea3e P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:51.111104 25199 leader_election.cc:302] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters: 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:51.111248 27808 raft_consensus.cc:2781] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:51.111332 27808 raft_consensus.cc:686] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:51.111452 27808 consensus_queue.cc:237] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:51.113463 22440 catalog_manager.cc:5484] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:51.140344 26995 raft_consensus.cc:979] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3: : Instructing follower 7db4178eed174225bfb24f5f9445ce88 to start an election
I20240502 07:58:51.140471 29689 raft_consensus.cc:1067] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 2 LEADER]: Signalling peer 7db4178eed174225bfb24f5f9445ce88 to start an election
I20240502 07:58:51.141319 22845 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "9d2283e5d28347519b38e876d840608e"
dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
from {username='slave'} at 127.16.85.130:34265
I20240502 07:58:51.141456 22845 raft_consensus.cc:482] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 07:58:51.141513 22845 raft_consensus.cc:3037] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 2 FOLLOWER]: Advancing to term 3
I20240502 07:58:51.142321 22845 raft_consensus.cc:504] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 3 FOLLOWER]: Starting forced leader election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:51.142843 22845 leader_election.cc:288] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 3 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:51.143317 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d2283e5d28347519b38e876d840608e" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 3 candidate_status { last_received { term: 2 index: 9 } } ignore_live_leader: true dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:51.143317 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d2283e5d28347519b38e876d840608e" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 3 candidate_status { last_received { term: 2 index: 9 } } ignore_live_leader: true dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:51.143435 24872 raft_consensus.cc:3037] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 2 FOLLOWER]: Advancing to term 3
I20240502 07:58:51.143435 25265 raft_consensus.cc:3037] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69 [term 2 FOLLOWER]: Advancing to term 3
I20240502 07:58:51.144166 29570 raft_consensus.cc:482] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 74f9aa91c2c448f980b512fe2e8d672e)
I20240502 07:58:51.144246 29570 raft_consensus.cc:504] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.144399 24872 raft_consensus.cc:2454] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 3.
I20240502 07:58:51.144399 25265 raft_consensus.cc:2454] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 3.
I20240502 07:58:51.144574 29570 leader_election.cc:288] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:51.145105 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "325b6a98112d4eef9488d7ef657bc375" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 2 candidate_status { last_received { term: 1 index: 1 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:51.145134 22783 leader_election.cc:302] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:51.145282 29773 raft_consensus.cc:2781] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 3 FOLLOWER]: Leader election won for term 3
I20240502 07:58:51.145340 29773 raft_consensus.cc:686] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 3 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:51.145455 29773 consensus_queue.cc:237] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 9, Current term: 3, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:51.146363 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d2283e5d28347519b38e876d840608e" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 3 candidate_status { last_received { term: 2 index: 9 } } ignore_live_leader: true dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:51.146482 22717 raft_consensus.cc:3032] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 2 LEADER]: Stepping down as leader of term 2
I20240502 07:58:51.146520 22717 raft_consensus.cc:729] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 2 LEADER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:51.146674 22717 consensus_queue.cc:260] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 9, Current term: 2, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:51.146872 22717 raft_consensus.cc:3037] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 2 FOLLOWER]: Advancing to term 3
I20240502 07:58:51.147770 22717 raft_consensus.cc:2454] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 3.
W20240502 07:58:51.149158 25070 leader_election.cc:334] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:51.149230 25070 leader_election.cc:302] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699; no voters: 74f9aa91c2c448f980b512fe2e8d672e, 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:51.149375 22440 catalog_manager.cc:5484] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 2 to 3, leader changed from 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 3 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:51.150091 29570 raft_consensus.cc:2726] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 07:58:51.154959 22845 raft_consensus.cc:1203] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 07:58:51.177174 27808 raft_consensus.cc:482] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 74f9aa91c2c448f980b512fe2e8d672e)
I20240502 07:58:51.177261 27808 raft_consensus.cc:504] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.177603 27808 leader_election.cc:288] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:51.177920 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "325b6a98112d4eef9488d7ef657bc375" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 2 candidate_status { last_received { term: 1 index: 1 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:51.178083 25134 raft_consensus.cc:2454] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:51.178321 25202 leader_election.cc:302] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:51.178467 27808 raft_consensus.cc:2781] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader pre-election won for term 2
W20240502 07:58:51.178500 25201 leader_election.cc:334] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:51.178519 27808 raft_consensus.cc:482] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 74f9aa91c2c448f980b512fe2e8d672e)
I20240502 07:58:51.178625 27808 raft_consensus.cc:3037] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Advancing to term 2
I20240502 07:58:51.179574 27808 raft_consensus.cc:504] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.179868 27808 leader_election.cc:288] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 2 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:51.180140 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "325b6a98112d4eef9488d7ef657bc375" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 2 candidate_status { last_received { term: 1 index: 1 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:51.180266 25134 raft_consensus.cc:3037] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Advancing to term 2
W20240502 07:58:51.181020 25201 leader_election.cc:334] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:51.181252 25134 raft_consensus.cc:2454] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 2.
I20240502 07:58:51.181483 25202 leader_election.cc:302] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7eced220cf1642f3873fc385b6191e69; no voters: 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:51.181658 27808 raft_consensus.cc:2781] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:51.181741 27808 raft_consensus.cc:686] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [term 2 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:51.181880 27808 consensus_queue.cc:237] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 1, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.183594 22435 catalog_manager.cc:5484] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 1 to 2, leader changed from 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 2 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:51.185055 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:51.212486 25265 raft_consensus.cc:1203] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
W20240502 07:58:51.244771 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:51.310194 27808 raft_consensus.cc:482] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:51.310298 27808 raft_consensus.cc:504] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:51.310706 27808 leader_election.cc:288] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:51.311021 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "41284d865ebb4cd18f2ce08f66343262" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:51.311168 24872 raft_consensus.cc:2454] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:51.311422 25199 leader_election.cc:302] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:51.311579 27808 raft_consensus.cc:2781] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
W20240502 07:58:51.311616 25201 leader_election.cc:334] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:51.311671 27808 raft_consensus.cc:482] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:51.311720 27808 raft_consensus.cc:3037] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:51.312498 27808 raft_consensus.cc:504] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:51.312824 27808 leader_election.cc:288] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:51.313094 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "41284d865ebb4cd18f2ce08f66343262" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:51.313210 24872 raft_consensus.cc:3037] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
W20240502 07:58:51.313856 25201 leader_election.cc:334] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:51.314237 24872 raft_consensus.cc:2454] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:51.314486 25199 leader_election.cc:302] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters: 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:51.314635 27808 raft_consensus.cc:2781] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:51.314721 27808 raft_consensus.cc:686] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:51.314895 27808 consensus_queue.cc:237] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:51.316304 25135 raft_consensus.cc:1261] T 28832d9e18ea41b194cf33d8c7971e23 P 4654231dfe994c1a9d2e37d9c3d75699 [term 2 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
I20240502 07:58:51.316604 29773 consensus_queue.cc:1035] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 07:58:51.316962 22440 catalog_manager.cc:5484] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } }
W20240502 07:58:51.411139 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 07:58:51.467334 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:51.472793 25135 raft_consensus.cc:1261] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 2 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 2 index: 3. (index mismatch)
I20240502 07:58:51.473126 27338 consensus_queue.cc:1035] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:51.475205 22717 raft_consensus.cc:3037] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Advancing to term 2
I20240502 07:58:51.476287 22717 raft_consensus.cc:1261] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3 [term 2 LEARNER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 2 index: 3. (index mismatch)
W20240502 07:58:51.476634 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:51.476642 27338 consensus_queue.cc:1035] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:51.478454 27338 raft_consensus.cc:1050] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537: attempting to promote NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 to VOTER
I20240502 07:58:51.478788 27338 consensus_queue.cc:237] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 2.3, Last appended by leader: 2, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false } }
I20240502 07:58:51.479990 25135 raft_consensus.cc:1261] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 2 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 2 index: 3. Preceding OpId from leader: term: 2 index: 4. (index mismatch)
I20240502 07:58:51.480373 22717 raft_consensus.cc:1261] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3 [term 2 LEARNER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 2 index: 3. Preceding OpId from leader: term: 2 index: 4. (index mismatch)
I20240502 07:58:51.480558 29934 consensus_queue.cc:1035] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:51.480963 29933 consensus_queue.cc:1035] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 07:58:51.481556 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:51.482364 29933 raft_consensus.cc:2932] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 [term 2 LEADER]: Committing config change with OpId 2.4: config changed from index 2 to 4, 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) changed from NON_VOTER to VOTER. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false } } }
I20240502 07:58:51.482705 22717 raft_consensus.cc:2932] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3 [term 2 FOLLOWER]: Committing config change with OpId 2.4: config changed from index 2 to 4, 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) changed from NON_VOTER to VOTER. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false } } }
I20240502 07:58:51.483847 25135 raft_consensus.cc:2932] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 2 FOLLOWER]: Committing config change with OpId 2.4: config changed from index 2 to 4, 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) changed from NON_VOTER to VOTER. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false } } }
I20240502 07:58:51.484896 22442 catalog_manager.cc:5484] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 2 to 4, 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) changed from NON_VOTER to VOTER. New cstate: current_term: 2 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: false } } }
I20240502 07:58:51.520933 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:51.523953 24872 raft_consensus.cc:1261] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:51.524293 27808 consensus_queue.cc:1035] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:58:51.524415 25201 consensus_peers.cc:485] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:51.526249 29937 tablet.cc:1722] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:51.526444 22440 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet 41284d865ebb4cd18f2ce08f66343262 (table rebalancer_dynamic_table [id=a295d5f8cc2c4b429ea679bc6e4a49cc]). Expected version 1 got 0
I20240502 07:58:51.526629 29937 tablet.cc:1620] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:51.527020 29938 tablet.cc:1722] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:51.527271 29938 tablet.cc:1620] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:51.527498 22440 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet 41284d865ebb4cd18f2ce08f66343262 (table rebalancer_dynamic_table [id=a295d5f8cc2c4b429ea679bc6e4a49cc]). Expected version 1 got 0
I20240502 07:58:51.530004 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=a295d5f8cc2c4b429ea679bc6e4a49cc] alter complete (version 1)
I20240502 07:58:51.550168 22717 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 3 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 3 index: 10. (index mismatch)
I20240502 07:58:51.550489 29773 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 07:58:51.576797 27495 consensus_queue.cc:1035] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:51.590399 25265 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69 [term 3 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 3 index: 10. (index mismatch)
I20240502 07:58:51.590700 29773 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 07:58:51.592648 24872 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 3 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 3 index: 10. (index mismatch)
I20240502 07:58:51.592990 29177 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 07:58:51.601425 27808 consensus_queue.cc:1035] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:58:51.604902 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:51.609756 28823 consensus_queue.cc:1035] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:51.631786 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:51.635351 22435 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_24"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:51.636960 29937 tablet.cc:1722] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:51.637229 29937 tablet.cc:1620] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:51.639649 29938 tablet.cc:1722] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:51.639897 29938 tablet.cc:1620] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:51.643162 24850 tablet_service.cc:1467] Processing CreateTablet for tablet ca94ddec3cd540c995b905cf07ab5cce (DEFAULT_TABLE table=rebalancer_extra_table_24 [id=6a8df810c5444053b08ce97f905b2c1b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:51.643163 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 1cabc294b14d4e7d99d4fb6d0024d5a0 (DEFAULT_TABLE table=rebalancer_extra_table_24 [id=6a8df810c5444053b08ce97f905b2c1b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:51.643196 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 45cbfa8bf4b94cfb8284fc7e1fb231a8 (DEFAULT_TABLE table=rebalancer_extra_table_24 [id=6a8df810c5444053b08ce97f905b2c1b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:51.643373 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ca94ddec3cd540c995b905cf07ab5cce. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:51.643373 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1cabc294b14d4e7d99d4fb6d0024d5a0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:51.643661 22566 tablet_service.cc:1467] Processing CreateTablet for tablet ca94ddec3cd540c995b905cf07ab5cce (DEFAULT_TABLE table=rebalancer_extra_table_24 [id=6a8df810c5444053b08ce97f905b2c1b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:51.643968 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ca94ddec3cd540c995b905cf07ab5cce. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:51.645597 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 1cabc294b14d4e7d99d4fb6d0024d5a0 (DEFAULT_TABLE table=rebalancer_extra_table_24 [id=6a8df810c5444053b08ce97f905b2c1b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:51.645665 29973 tablet_bootstrap.cc:492] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:51.645821 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1cabc294b14d4e7d99d4fb6d0024d5a0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:51.645895 29974 tablet_bootstrap.cc:492] T ca94ddec3cd540c995b905cf07ab5cce P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:51.646484 29973 tablet_bootstrap.cc:654] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:51.646689 29974 tablet_bootstrap.cc:654] T ca94ddec3cd540c995b905cf07ab5cce P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:51.646970 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 45cbfa8bf4b94cfb8284fc7e1fb231a8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:51.647382 29975 tablet_bootstrap.cc:492] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:51.647583 29974 tablet_bootstrap.cc:492] T ca94ddec3cd540c995b905cf07ab5cce P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:51.647658 29974 ts_tablet_manager.cc:1367] T ca94ddec3cd540c995b905cf07ab5cce P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:51.648037 29974 raft_consensus.cc:348] T ca94ddec3cd540c995b905cf07ab5cce P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:51.648128 29974 raft_consensus.cc:374] T ca94ddec3cd540c995b905cf07ab5cce P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:51.648156 29974 raft_consensus.cc:729] T ca94ddec3cd540c995b905cf07ab5cce P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:51.648253 29974 consensus_queue.cc:260] T ca94ddec3cd540c995b905cf07ab5cce P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:51.648360 29975 tablet_bootstrap.cc:654] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:51.648487 29974 ts_tablet_manager.cc:1398] T ca94ddec3cd540c995b905cf07ab5cce P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:51.648612 29974 tablet_bootstrap.cc:492] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:51.649266 29973 tablet_bootstrap.cc:492] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:51.649325 29973 ts_tablet_manager.cc:1367] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:51.649365 29974 tablet_bootstrap.cc:654] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:51.650579 25135 raft_consensus.cc:1261] T 325b6a98112d4eef9488d7ef657bc375 P 4654231dfe994c1a9d2e37d9c3d75699 [term 2 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 2 index: 2. (index mismatch)
I20240502 07:58:51.650696 29974 tablet_bootstrap.cc:492] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:51.650765 29974 ts_tablet_manager.cc:1367] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:51.650897 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 45cbfa8bf4b94cfb8284fc7e1fb231a8 (DEFAULT_TABLE table=rebalancer_extra_table_24 [id=6a8df810c5444053b08ce97f905b2c1b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:51.651154 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 45cbfa8bf4b94cfb8284fc7e1fb231a8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:51.651096 29974 raft_consensus.cc:348] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:51.651137 29941 consensus_queue.cc:1035] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 07:58:51.651196 29974 raft_consensus.cc:374] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:51.651223 29974 raft_consensus.cc:729] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:51.651319 29974 consensus_queue.cc:260] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:51.651597 29974 ts_tablet_manager.cc:1398] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:51.652040 22435 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=a295d5f8cc2c4b429ea679bc6e4a49cc] alter complete (version 2)
I20240502 07:58:51.652491 25246 tablet_service.cc:1467] Processing CreateTablet for tablet ca94ddec3cd540c995b905cf07ab5cce (DEFAULT_TABLE table=rebalancer_extra_table_24 [id=6a8df810c5444053b08ce97f905b2c1b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:51.652491 25245 tablet_service.cc:1467] Processing CreateTablet for tablet 1cabc294b14d4e7d99d4fb6d0024d5a0 (DEFAULT_TABLE table=rebalancer_extra_table_24 [id=6a8df810c5444053b08ce97f905b2c1b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:51.652674 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 45cbfa8bf4b94cfb8284fc7e1fb231a8 (DEFAULT_TABLE table=rebalancer_extra_table_24 [id=6a8df810c5444053b08ce97f905b2c1b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:51.652781 25245 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1cabc294b14d4e7d99d4fb6d0024d5a0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:51.652884 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 45cbfa8bf4b94cfb8284fc7e1fb231a8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:51.653247 29973 raft_consensus.cc:348] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:51.653332 29973 raft_consensus.cc:374] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:51.653357 29973 raft_consensus.cc:729] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:51.653474 29973 consensus_queue.cc:260] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:51.653892 29973 ts_tablet_manager.cc:1398] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:51.654142 29973 tablet_bootstrap.cc:492] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:51.654788 29983 tablet_bootstrap.cc:492] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:51.654956 29973 tablet_bootstrap.cc:654] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:51.655607 29983 tablet_bootstrap.cc:654] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:51.655862 29981 tablet_bootstrap.cc:492] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:51.657514 29981 tablet_bootstrap.cc:654] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:51.658327 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ca94ddec3cd540c995b905cf07ab5cce. 1 dirs total, 0 dirs full, 0 dirs failed
W20240502 07:58:51.658907 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:51.661149 29973 tablet_bootstrap.cc:492] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:51.661223 29973 ts_tablet_manager.cc:1367] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
I20240502 07:58:51.661566 29973 raft_consensus.cc:348] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.661661 29973 raft_consensus.cc:374] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:51.661693 29973 raft_consensus.cc:729] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:51.661788 29973 consensus_queue.cc:260] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.662010 29973 ts_tablet_manager.cc:1398] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:51.662431 29975 tablet_bootstrap.cc:492] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:51.662505 29975 ts_tablet_manager.cc:1367] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.015s user 0.000s sys 0.001s
I20240502 07:58:51.662926 29975 raft_consensus.cc:348] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:51.663019 29975 raft_consensus.cc:374] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:51.663051 29975 raft_consensus.cc:729] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:51.663151 29975 consensus_queue.cc:260] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
W20240502 07:58:51.663194 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:51.663410 29975 ts_tablet_manager.cc:1398] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:51.664258 29981 tablet_bootstrap.cc:492] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:51.664405 29981 ts_tablet_manager.cc:1367] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.009s user 0.002s sys 0.000s
I20240502 07:58:51.664512 29983 tablet_bootstrap.cc:492] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:51.664697 29983 ts_tablet_manager.cc:1367] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.000s
I20240502 07:58:51.664938 29981 raft_consensus.cc:348] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.665091 29981 raft_consensus.cc:374] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:51.665155 29981 raft_consensus.cc:729] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:51.665314 29981 consensus_queue.cc:260] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.665357 29983 raft_consensus.cc:348] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:51.665511 29983 raft_consensus.cc:374] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:51.665556 29983 raft_consensus.cc:729] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:51.665596 29981 ts_tablet_manager.cc:1398] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:51.665702 29983 consensus_queue.cc:260] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:51.666548 29983 ts_tablet_manager.cc:1398] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:51.666734 29983 tablet_bootstrap.cc:492] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:51.667752 29983 tablet_bootstrap.cc:654] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:51.669368 29983 tablet_bootstrap.cc:492] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:51.669445 29983 ts_tablet_manager.cc:1367] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:51.669801 29983 raft_consensus.cc:348] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.669889 29983 raft_consensus.cc:374] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:51.669939 29983 raft_consensus.cc:729] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:51.670038 29983 consensus_queue.cc:260] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.670279 29983 ts_tablet_manager.cc:1398] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:51.670420 29983 tablet_bootstrap.cc:492] T ca94ddec3cd540c995b905cf07ab5cce P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:51.671146 29983 tablet_bootstrap.cc:654] T ca94ddec3cd540c995b905cf07ab5cce P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:51.672048 29983 tablet_bootstrap.cc:492] T ca94ddec3cd540c995b905cf07ab5cce P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:51.672119 29983 ts_tablet_manager.cc:1367] T ca94ddec3cd540c995b905cf07ab5cce P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:51.672464 29983 raft_consensus.cc:348] T ca94ddec3cd540c995b905cf07ab5cce P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:51.672542 29983 raft_consensus.cc:374] T ca94ddec3cd540c995b905cf07ab5cce P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:51.672569 29983 raft_consensus.cc:729] T ca94ddec3cd540c995b905cf07ab5cce P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:51.672686 29983 consensus_queue.cc:260] T ca94ddec3cd540c995b905cf07ab5cce P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:51.672909 29983 ts_tablet_manager.cc:1398] T ca94ddec3cd540c995b905cf07ab5cce P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:51.708721 27808 raft_consensus.cc:482] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:51.708789 27808 raft_consensus.cc:504] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.709066 27808 leader_election.cc:288] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:51.709235 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "45cbfa8bf4b94cfb8284fc7e1fb231a8" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:51.709336 22979 raft_consensus.cc:2454] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:51.709378 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "45cbfa8bf4b94cfb8284fc7e1fb231a8" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:51.709483 25135 raft_consensus.cc:2454] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:51.709642 25200 leader_election.cc:302] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:51.712921 27808 raft_consensus.cc:2781] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:51.712965 27808 raft_consensus.cc:482] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:51.713003 27808 raft_consensus.cc:3037] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:51.713680 27808 raft_consensus.cc:504] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.713992 27808 leader_election.cc:288] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:51.714258 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "45cbfa8bf4b94cfb8284fc7e1fb231a8" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:51.714350 25135 raft_consensus.cc:3037] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:51.715085 25135 raft_consensus.cc:2454] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:51.715251 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "45cbfa8bf4b94cfb8284fc7e1fb231a8" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:51.715330 22979 raft_consensus.cc:3037] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:51.715315 25202 leader_election.cc:302] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:51.715437 27808 raft_consensus.cc:2781] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:51.715502 27808 raft_consensus.cc:686] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:51.715593 27808 consensus_queue.cc:237] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.716127 22979 raft_consensus.cc:2454] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:51.726061 22442 catalog_manager.cc:5484] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:51.727624 29941 raft_consensus.cc:482] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:51.727699 29941 raft_consensus.cc:504] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:51.727979 29941 leader_election.cc:288] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:51.728415 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0d3af4df9b410baaa088c744eeb2a6" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:51.728523 22717 raft_consensus.cc:2454] T cf0d3af4df9b410baaa088c744eeb2a6 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:51.728747 25199 leader_election.cc:302] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:51.728874 29941 raft_consensus.cc:2781] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20240502 07:58:51.728914 29941 raft_consensus.cc:482] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:51.728948 29941 raft_consensus.cc:3037] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Advancing to term 2
W20240502 07:58:51.729058 25201 leader_election.cc:334] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:51.729694 29941 raft_consensus.cc:504] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:51.729988 29941 leader_election.cc:288] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 2 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:51.730317 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0d3af4df9b410baaa088c744eeb2a6" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:51.730417 22717 raft_consensus.cc:3037] T cf0d3af4df9b410baaa088c744eeb2a6 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Advancing to term 2
W20240502 07:58:51.730758 25201 leader_election.cc:334] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:51.731302 22717 raft_consensus.cc:2454] T cf0d3af4df9b410baaa088c744eeb2a6 P 725969d490a04fdfb3ba890c3f5096e3 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 2.
I20240502 07:58:51.731518 25199 leader_election.cc:302] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7eced220cf1642f3873fc385b6191e69; no voters: 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:51.731669 29941 raft_consensus.cc:2781] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 07:58:51.731742 29941 raft_consensus.cc:686] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [term 2 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:51.731839 29941 consensus_queue.cc:237] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:51.733376 22442 catalog_manager.cc:5484] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 1 to 2, leader changed from 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 2 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:51.755234 29882 raft_consensus.cc:1050] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 7eced220cf1642f3873fc385b6191e69 to VOTER
I20240502 07:58:51.755522 29882 consensus_queue.cc:237] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:51.756466 25265 raft_consensus.cc:1261] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:51.756510 22845 raft_consensus.cc:1261] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:51.756811 30009 consensus_queue.cc:1035] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:51.756873 24870 raft_consensus.cc:1261] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 07:58:51.757162 30009 consensus_queue.cc:1035] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:51.757292 30007 consensus_queue.cc:1035] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 07:58:51.758927 30007 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:51.759331 22845 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:51.759449 24870 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:51.761546 22435 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:51.761576 25265 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:51.761950 22442 catalog_manager.cc:5484] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 7 to 8, 7eced220cf1642f3873fc385b6191e69 (127.16.85.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:51.765275 22979 consensus_queue.cc:237] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:51.766175 25266 raft_consensus.cc:1261] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 07:58:51.766526 24872 raft_consensus.cc:1261] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 07:58:51.766880 30009 consensus_queue.cc:1035] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:51.767303 30007 consensus_queue.cc:1035] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 07:58:51.768378 30009 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:51.768793 29937 tablet.cc:1722] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:51.768906 25266 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:51.769109 29938 tablet.cc:1722] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:51.769953 22429 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e5d5f67e026344998d5fb079f532d076 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:51.772480 24872 raft_consensus.cc:2932] T e5d5f67e026344998d5fb079f532d076 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:51.773741 22442 catalog_manager.cc:5484] T e5d5f67e026344998d5fb079f532d076 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 8 to 9, VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) evicted. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:51.774771 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=a295d5f8cc2c4b429ea679bc6e4a49cc] alter complete (version 3)
I20240502 07:58:51.776914 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet e5d5f67e026344998d5fb079f532d076 with delete_type TABLET_DATA_TOMBSTONED (TS 7db4178eed174225bfb24f5f9445ce88 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:51.777410 30015 tablet_replica.cc:330] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:51.777561 30015 raft_consensus.cc:2229] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:51.777841 30015 raft_consensus.cc:2258] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:51.777977 30010 raft_consensus.cc:1050] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f: attempting to promote NON_VOTER 7db4178eed174225bfb24f5f9445ce88 to VOTER
I20240502 07:58:51.778319 30010 consensus_queue.cc:237] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } }
I20240502 07:58:51.778441 30015 ts_tablet_manager.cc:1875] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:51.779349 24872 raft_consensus.cc:1261] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:51.779467 22845 raft_consensus.cc:1261] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:51.779455 22586 raft_consensus.cc:1261] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 07:58:51.779681 29882 consensus_queue.cc:1035] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:51.779979 29882 consensus_queue.cc:1035] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:51.780160 30007 consensus_queue.cc:1035] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 07:58:51.780534 30015 ts_tablet_manager.cc:1888] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 07:58:51.780632 30015 log.cc:1196] T e5d5f67e026344998d5fb079f532d076 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/e5d5f67e026344998d5fb079f532d076
I20240502 07:58:51.781147 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet e5d5f67e026344998d5fb079f532d076 (table rebalance_test_table_1 [id=ed9ff98b498847f8a18550017ae8fc8c]) successfully deleted
I20240502 07:58:51.781904 29882 raft_consensus.cc:2932] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:51.782282 24872 raft_consensus.cc:2932] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:51.782367 22586 raft_consensus.cc:2932] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:51.784126 22436 catalog_manager.cc:5484] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:51.784216 22845 raft_consensus.cc:2932] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { replace: true } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:51.789131 22979 consensus_queue.cc:237] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } }
I20240502 07:58:51.789907 22845 raft_consensus.cc:1261] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:51.789947 22586 raft_consensus.cc:1261] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 07:58:51.790165 30007 consensus_queue.cc:1035] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:51.790285 30010 consensus_queue.cc:1035] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 07:58:51.791456 30009 raft_consensus.cc:2932] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:51.791891 22586 raft_consensus.cc:2932] T 6af67c65672f45a68e960936351e223f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:51.791909 22845 raft_consensus.cc:2932] T 6af67c65672f45a68e960936351e223f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } }
I20240502 07:58:51.792855 22429 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6af67c65672f45a68e960936351e223f with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:51.793633 22440 catalog_manager.cc:5484] T 6af67c65672f45a68e960936351e223f P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 3 to 4, VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) evicted. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 07:58:51.795552 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet 6af67c65672f45a68e960936351e223f with delete_type TABLET_DATA_TOMBSTONED (TS 6d02017318a2461796d431962fe6a537 not found in new config with opid_index 4) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:51.795966 30020 tablet_replica.cc:330] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:51.796092 30020 raft_consensus.cc:2229] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:51.796222 30020 raft_consensus.cc:2258] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:51.796588 30020 ts_tablet_manager.cc:1875] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:51.798316 30020 ts_tablet_manager.cc:1888] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.3
I20240502 07:58:51.798373 30020 log.cc:1196] T 6af67c65672f45a68e960936351e223f P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/6af67c65672f45a68e960936351e223f
I20240502 07:58:51.798847 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet 6af67c65672f45a68e960936351e223f (table rebalancer_extra_table_8 [id=3eca1e0de44446a9b1e8bb318b02389a]) successfully deleted
I20240502 07:58:51.829080 30006 raft_consensus.cc:482] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:51.829195 30006 raft_consensus.cc:504] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:51.829526 30006 leader_election.cc:288] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:51.829797 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca94ddec3cd540c995b905cf07ab5cce" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:51.829824 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca94ddec3cd540c995b905cf07ab5cce" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:51.829910 24872 raft_consensus.cc:2454] T ca94ddec3cd540c995b905cf07ab5cce P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:51.829933 25266 raft_consensus.cc:2454] T ca94ddec3cd540c995b905cf07ab5cce P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:51.830169 22521 leader_election.cc:302] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7eced220cf1642f3873fc385b6191e69, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:51.830344 30006 raft_consensus.cc:2781] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:51.830411 30006 raft_consensus.cc:482] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:51.830435 30006 raft_consensus.cc:3037] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:51.831494 30006 raft_consensus.cc:504] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:51.831817 30006 leader_election.cc:288] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:51.832087 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca94ddec3cd540c995b905cf07ab5cce" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:51.832130 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca94ddec3cd540c995b905cf07ab5cce" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:51.832206 25266 raft_consensus.cc:3037] T ca94ddec3cd540c995b905cf07ab5cce P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:51.832206 24872 raft_consensus.cc:3037] T ca94ddec3cd540c995b905cf07ab5cce P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:51.832506 29934 raft_consensus.cc:482] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:51.832587 29934 raft_consensus.cc:504] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:51.832919 29934 leader_election.cc:288] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:51.833026 24872 raft_consensus.cc:2454] T ca94ddec3cd540c995b905cf07ab5cce P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:51.833027 25266 raft_consensus.cc:2454] T ca94ddec3cd540c995b905cf07ab5cce P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:51.833233 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cabc294b14d4e7d99d4fb6d0024d5a0" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:51.833278 22519 leader_election.cc:302] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:51.833334 25266 raft_consensus.cc:2454] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:51.833354 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cabc294b14d4e7d99d4fb6d0024d5a0" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:51.833473 25135 raft_consensus.cc:2454] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:51.833541 24807 leader_election.cc:302] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:51.833688 29934 raft_consensus.cc:2781] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:51.833772 29934 raft_consensus.cc:482] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:51.833823 29934 raft_consensus.cc:3037] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:51.833859 24714 raft_consensus.cc:2781] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:51.833942 24714 raft_consensus.cc:686] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:51.834069 24714 consensus_queue.cc:237] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:51.834741 29934 raft_consensus.cc:504] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:51.835088 29934 leader_election.cc:288] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:51.835450 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cabc294b14d4e7d99d4fb6d0024d5a0" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:51.835505 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cabc294b14d4e7d99d4fb6d0024d5a0" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:51.835553 25135 raft_consensus.cc:3037] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:51.835579 25266 raft_consensus.cc:3037] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:51.836169 22444 catalog_manager.cc:5484] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:51.836464 25266 raft_consensus.cc:2454] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:51.836467 25135 raft_consensus.cc:2454] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:51.836763 24808 leader_election.cc:302] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:51.836946 29934 raft_consensus.cc:2781] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:51.837016 29934 raft_consensus.cc:686] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:58:51.837113 29934 consensus_queue.cc:237] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:51.838860 22444 catalog_manager.cc:5484] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:51.876983 22444 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:51.877089 22444 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:51.878719 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 41284d865ebb4cd18f2ce08f66343262
I20240502 07:58:51.879175 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet 41284d865ebb4cd18f2ce08f66343262 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:51 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:51.879328 30020 tablet_replica.cc:330] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:51.879442 30020 raft_consensus.cc:2229] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:51.879568 30020 raft_consensus.cc:2258] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:51.879617 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet 41284d865ebb4cd18f2ce08f66343262 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:51 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:51.880021 30024 tablet_replica.cc:330] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:51.880088 30020 ts_tablet_manager.cc:1875] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:51.880182 30024 raft_consensus.cc:2229] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:51.880362 30024 raft_consensus.cc:2258] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:51.880834 30024 ts_tablet_manager.cc:1875] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:51.881979 30020 ts_tablet_manager.cc:1888] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:51.882059 30020 log.cc:1196] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/41284d865ebb4cd18f2ce08f66343262
I20240502 07:58:51.882298 30020 ts_tablet_manager.cc:1909] T 41284d865ebb4cd18f2ce08f66343262 P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:51.882858 30024 ts_tablet_manager.cc:1888] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:51.882927 30024 log.cc:1196] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/41284d865ebb4cd18f2ce08f66343262
I20240502 07:58:51.882973 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet 41284d865ebb4cd18f2ce08f66343262 (table rebalancer_dynamic_table_new_name [id=a295d5f8cc2c4b429ea679bc6e4a49cc]) successfully deleted
I20240502 07:58:51.883150 30024 ts_tablet_manager.cc:1909] T 41284d865ebb4cd18f2ce08f66343262 P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:58:51.883773 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet 41284d865ebb4cd18f2ce08f66343262 (table rebalancer_dynamic_table_new_name [id=a295d5f8cc2c4b429ea679bc6e4a49cc]) successfully deleted
W20240502 07:58:51.960918 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_DELETED RPC failed for tablet 41284d865ebb4cd18f2ce08f66343262: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:51.980239 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
W20240502 07:58:51.983916 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 07:58:51.985327 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 2a8935d073c74e9a80635b88d9fb0a6f (DEFAULT_TABLE table=rebalancer_dynamic_table [id=0a9339f7788649aea19993a66e360b02]), partition=
I20240502 07:58:51.985327 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 2a8935d073c74e9a80635b88d9fb0a6f (DEFAULT_TABLE table=rebalancer_dynamic_table [id=0a9339f7788649aea19993a66e360b02]), partition=
I20240502 07:58:51.985538 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2a8935d073c74e9a80635b88d9fb0a6f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:51.985540 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2a8935d073c74e9a80635b88d9fb0a6f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:51.987427 29973 tablet_bootstrap.cc:492] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:51.988233 29973 tablet_bootstrap.cc:654] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:51.988713 29981 tablet_bootstrap.cc:492] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:51.989113 29973 tablet_bootstrap.cc:492] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:51.989187 29973 ts_tablet_manager.cc:1367] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:51.989420 29981 tablet_bootstrap.cc:654] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:51.989490 29973 raft_consensus.cc:348] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.989558 29973 raft_consensus.cc:374] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:51.989576 29973 raft_consensus.cc:729] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:51.989655 29973 consensus_queue.cc:260] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.989879 29973 ts_tablet_manager.cc:1398] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:51.990286 29981 tablet_bootstrap.cc:492] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:51.990360 29981 ts_tablet_manager.cc:1367] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:51.990711 29981 raft_consensus.cc:348] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.990805 29981 raft_consensus.cc:374] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:51.990832 29981 raft_consensus.cc:729] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:51.990957 29981 consensus_queue.cc:260] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:51.991181 29981 ts_tablet_manager.cc:1398] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
W20240502 07:58:52.009922 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 07:58:52.063815 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:52.130383 22845 consensus_queue.cc:237] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 3.10, Last appended by leader: 9, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } }
I20240502 07:58:52.131953 24872 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 3 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 3 index: 10. Preceding OpId from leader: term: 3 index: 11. (index mismatch)
I20240502 07:58:52.132306 25266 raft_consensus.cc:1261] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69 [term 3 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 3 index: 10. Preceding OpId from leader: term: 3 index: 11. (index mismatch)
I20240502 07:58:52.132377 28408 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 07:58:52.132755 28408 consensus_queue.cc:1035] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 07:58:52.134011 29773 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 [term 3 LEADER]: Committing config change with OpId 3.11: config changed from index 9 to 11, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:52.134464 22717 raft_consensus.cc:1261] T cf0d3af4df9b410baaa088c744eeb2a6 P 725969d490a04fdfb3ba890c3f5096e3 [term 2 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 2 index: 1. (index mismatch)
I20240502 07:58:52.134441 24872 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 6d02017318a2461796d431962fe6a537 [term 3 FOLLOWER]: Committing config change with OpId 3.11: config changed from index 9 to 11, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:52.135939 29941 consensus_queue.cc:1035] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:52.135941 25266 raft_consensus.cc:2932] T 9d2283e5d28347519b38e876d840608e P 7eced220cf1642f3873fc385b6191e69 [term 3 FOLLOWER]: Committing config change with OpId 3.11: config changed from index 9 to 11, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } }
I20240502 07:58:52.136368 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9d2283e5d28347519b38e876d840608e with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 07:58:52.137148 22442 catalog_manager.cc:5484] T 9d2283e5d28347519b38e876d840608e P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 9 to 11, VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) evicted. New cstate: current_term: 3 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
W20240502 07:58:52.140808 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:52.141237 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet 9d2283e5d28347519b38e876d840608e with delete_type TABLET_DATA_TOMBSTONED (TS 725969d490a04fdfb3ba890c3f5096e3 not found in new config with opid_index 11) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:52.141481 30110 tablet_replica.cc:330] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:52.141606 30110 raft_consensus.cc:2229] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 3 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:52.141752 30110 raft_consensus.cc:2258] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3 [term 3 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:52.142355 30110 ts_tablet_manager.cc:1875] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 07:58:52.143518 27808 consensus_queue.cc:1035] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:52.146607 30110 ts_tablet_manager.cc:1888] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 3.10
I20240502 07:58:52.147038 30110 log.cc:1196] T 9d2283e5d28347519b38e876d840608e P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/9d2283e5d28347519b38e876d840608e
I20240502 07:58:52.148483 27808 consensus_queue.cc:1035] T 45cbfa8bf4b94cfb8284fc7e1fb231a8 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:52.149998 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 9d2283e5d28347519b38e876d840608e (table rebalance_test_table_11 [id=0c1156cac15d48028ebf5c07c67ea832]) successfully deleted
I20240502 07:58:52.236037 30007 raft_consensus.cc:482] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:52.236132 30007 raft_consensus.cc:504] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:52.236439 30007 leader_election.cc:288] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:52.236743 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2a8935d073c74e9a80635b88d9fb0a6f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:52.236861 25135 raft_consensus.cc:2454] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:52.237269 22915 leader_election.cc:302] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 4654231dfe994c1a9d2e37d9c3d75699; no voters:
I20240502 07:58:52.237471 30007 raft_consensus.cc:2781] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:52.237522 30007 raft_consensus.cc:482] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
W20240502 07:58:52.237558 22914 leader_election.cc:334] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:52.237558 30007 raft_consensus.cc:3037] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:52.238615 30007 raft_consensus.cc:504] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:52.238886 30007 leader_election.cc:288] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:52.239159 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2a8935d073c74e9a80635b88d9fb0a6f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:52.239287 25135 raft_consensus.cc:3037] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:52.240149 25135 raft_consensus.cc:2454] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:52.240379 22915 leader_election.cc:302] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 4654231dfe994c1a9d2e37d9c3d75699; no voters:
W20240502 07:58:52.240563 22914 leader_election.cc:334] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:52.240778 30007 raft_consensus.cc:2781] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:52.240859 30007 raft_consensus.cc:686] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:52.240994 30007 consensus_queue.cc:237] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:52.242897 22444 catalog_manager.cc:5484] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:52.267129 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 07:58:52.273020 30006 consensus_queue.cc:1035] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:52.276304 30006 consensus_queue.cc:1035] T ca94ddec3cd540c995b905cf07ab5cce P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:58:52.298324 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 07:58:52.342998 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_26"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:52.348356 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 355e38f2928541cb851924f66cb5550e (DEFAULT_TABLE table=rebalancer_extra_table_26 [id=5d10795a208b43fdab1ef026bb539871]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:52.348697 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 355e38f2928541cb851924f66cb5550e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:52.348858 24852 tablet_service.cc:1467] Processing CreateTablet for tablet f1862e3480644235adaf4a58f2c45bd9 (DEFAULT_TABLE table=rebalancer_extra_table_26 [id=5d10795a208b43fdab1ef026bb539871]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:52.349110 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f1862e3480644235adaf4a58f2c45bd9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:52.349251 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 355e38f2928541cb851924f66cb5550e (DEFAULT_TABLE table=rebalancer_extra_table_26 [id=5d10795a208b43fdab1ef026bb539871]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:52.349478 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 355e38f2928541cb851924f66cb5550e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:52.351027 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 5b7fbabf24b240879fe8f15bcfa4b377 (DEFAULT_TABLE table=rebalancer_extra_table_26 [id=5d10795a208b43fdab1ef026bb539871]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:52.351090 30184 tablet_bootstrap.cc:492] T f1862e3480644235adaf4a58f2c45bd9 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:52.351214 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5b7fbabf24b240879fe8f15bcfa4b377. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:52.351473 30185 tablet_bootstrap.cc:492] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:52.351711 29981 tablet_bootstrap.cc:492] T 355e38f2928541cb851924f66cb5550e P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:52.352118 30184 tablet_bootstrap.cc:654] T f1862e3480644235adaf4a58f2c45bd9 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:52.352260 30185 tablet_bootstrap.cc:654] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:52.352517 29981 tablet_bootstrap.cc:654] T 355e38f2928541cb851924f66cb5550e P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:52.353374 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 355e38f2928541cb851924f66cb5550e (DEFAULT_TABLE table=rebalancer_extra_table_26 [id=5d10795a208b43fdab1ef026bb539871]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:52.353616 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 355e38f2928541cb851924f66cb5550e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:52.354123 30184 tablet_bootstrap.cc:492] T f1862e3480644235adaf4a58f2c45bd9 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:52.354200 30184 ts_tablet_manager.cc:1367] T f1862e3480644235adaf4a58f2c45bd9 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:52.354646 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 5b7fbabf24b240879fe8f15bcfa4b377 (DEFAULT_TABLE table=rebalancer_extra_table_26 [id=5d10795a208b43fdab1ef026bb539871]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:52.354641 30184 raft_consensus.cc:348] T f1862e3480644235adaf4a58f2c45bd9 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:52.354748 30184 raft_consensus.cc:374] T f1862e3480644235adaf4a58f2c45bd9 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:52.354786 30184 raft_consensus.cc:729] T f1862e3480644235adaf4a58f2c45bd9 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:52.354882 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5b7fbabf24b240879fe8f15bcfa4b377. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:52.354939 30184 consensus_queue.cc:260] T f1862e3480644235adaf4a58f2c45bd9 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:52.355244 30184 ts_tablet_manager.cc:1398] T f1862e3480644235adaf4a58f2c45bd9 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:52.355325 29981 tablet_bootstrap.cc:492] T 355e38f2928541cb851924f66cb5550e P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:52.355381 30184 tablet_bootstrap.cc:492] T 5b7fbabf24b240879fe8f15bcfa4b377 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:52.355384 29981 ts_tablet_manager.cc:1367] T 355e38f2928541cb851924f66cb5550e P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:52.355782 29981 raft_consensus.cc:348] T 355e38f2928541cb851924f66cb5550e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:52.355775 25115 tablet_service.cc:1467] Processing CreateTablet for tablet f1862e3480644235adaf4a58f2c45bd9 (DEFAULT_TABLE table=rebalancer_extra_table_26 [id=5d10795a208b43fdab1ef026bb539871]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:52.355868 29981 raft_consensus.cc:374] T 355e38f2928541cb851924f66cb5550e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:52.355896 29981 raft_consensus.cc:729] T 355e38f2928541cb851924f66cb5550e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:52.356040 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f1862e3480644235adaf4a58f2c45bd9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:52.355991 29981 consensus_queue.cc:260] T 355e38f2928541cb851924f66cb5550e P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:52.356107 30184 tablet_bootstrap.cc:654] T 5b7fbabf24b240879fe8f15bcfa4b377 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:52.356276 29981 ts_tablet_manager.cc:1398] T 355e38f2928541cb851924f66cb5550e P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:52.356900 30191 tablet_bootstrap.cc:492] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:52.357502 30184 tablet_bootstrap.cc:492] T 5b7fbabf24b240879fe8f15bcfa4b377 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:52.357558 30184 ts_tablet_manager.cc:1367] T 5b7fbabf24b240879fe8f15bcfa4b377 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:52.357834 29973 tablet_bootstrap.cc:492] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:52.357918 30184 raft_consensus.cc:348] T 5b7fbabf24b240879fe8f15bcfa4b377 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:52.358026 30184 raft_consensus.cc:374] T 5b7fbabf24b240879fe8f15bcfa4b377 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:52.358053 30184 raft_consensus.cc:729] T 5b7fbabf24b240879fe8f15bcfa4b377 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:52.358147 30184 consensus_queue.cc:260] T 5b7fbabf24b240879fe8f15bcfa4b377 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:52.358371 30184 ts_tablet_manager.cc:1398] T 5b7fbabf24b240879fe8f15bcfa4b377 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:52.358644 29973 tablet_bootstrap.cc:654] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:52.359252 30191 tablet_bootstrap.cc:654] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:52.359709 30185 tablet_bootstrap.cc:492] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:52.359825 30185 ts_tablet_manager.cc:1367] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.008s user 0.000s sys 0.002s
I20240502 07:58:52.360352 30185 raft_consensus.cc:348] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:52.360824 30189 tablet_bootstrap.cc:492] T 355e38f2928541cb851924f66cb5550e P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:52.361013 30191 tablet_bootstrap.cc:492] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:52.360481 30185 raft_consensus.cc:374] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:52.361232 30185 raft_consensus.cc:729] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:52.361598 30185 consensus_queue.cc:260] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:52.362444 30189 tablet_bootstrap.cc:654] T 355e38f2928541cb851924f66cb5550e P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:52.362778 30185 ts_tablet_manager.cc:1398] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.003s user 0.000s sys 0.001s
I20240502 07:58:52.363865 29973 tablet_bootstrap.cc:492] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:52.363950 29973 ts_tablet_manager.cc:1367] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:58:52.364291 29973 raft_consensus.cc:348] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:52.364378 29973 raft_consensus.cc:374] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:52.364421 29973 raft_consensus.cc:729] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:52.364552 29973 consensus_queue.cc:260] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:52.364856 29973 ts_tablet_manager.cc:1398] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:52.365429 30191 ts_tablet_manager.cc:1367] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:52.366500 30191 raft_consensus.cc:348] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:52.366585 30191 raft_consensus.cc:374] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:52.366616 30191 raft_consensus.cc:729] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:52.366735 30191 consensus_queue.cc:260] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:52.366972 30191 ts_tablet_manager.cc:1398] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:52.367763 28823 consensus_queue.cc:1035] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:52.371598 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:52.374234 28823 consensus_queue.cc:1035] T 1cabc294b14d4e7d99d4fb6d0024d5a0 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:52.375689 25133 raft_consensus.cc:1261] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
W20240502 07:58:52.376083 22914 consensus_peers.cc:485] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:52.376168 30007 consensus_queue.cc:1035] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:52.379159 22442 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 2a8935d073c74e9a80635b88d9fb0a6f (table rebalancer_dynamic_table [id=0a9339f7788649aea19993a66e360b02]). Expected version 1 got 0
I20240502 07:58:52.385339 30189 tablet_bootstrap.cc:492] T 355e38f2928541cb851924f66cb5550e P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:52.385421 30189 ts_tablet_manager.cc:1367] T 355e38f2928541cb851924f66cb5550e P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.025s user 0.002s sys 0.000s
I20240502 07:58:52.385821 30189 raft_consensus.cc:348] T 355e38f2928541cb851924f66cb5550e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:52.385910 30189 raft_consensus.cc:374] T 355e38f2928541cb851924f66cb5550e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:52.385933 30189 raft_consensus.cc:729] T 355e38f2928541cb851924f66cb5550e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:52.386036 30112 mvcc.cc:205] Tried to move back new op lower bound from 7023152055807655936 to 7023152055260307456. Current Snapshot: MvccSnapshot[applied={T|T < 7023152055807655936}]
I20240502 07:58:52.386693 30189 consensus_queue.cc:260] T 355e38f2928541cb851924f66cb5550e P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:52.387317 22442 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet 2a8935d073c74e9a80635b88d9fb0a6f (table rebalancer_dynamic_table [id=0a9339f7788649aea19993a66e360b02]). Expected version 1 got 0
I20240502 07:58:52.387604 30189 ts_tablet_manager.cc:1398] T 355e38f2928541cb851924f66cb5550e P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:52.388167 22444 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 2a8935d073c74e9a80635b88d9fb0a6f (table rebalancer_dynamic_table [id=0a9339f7788649aea19993a66e360b02]). Expected version 1 got 0
I20240502 07:58:52.388978 30200 tablet.cc:1722] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:52.389284 30200 tablet.cc:1620] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:52.393203 22442 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=0a9339f7788649aea19993a66e360b02] alter complete (version 1)
I20240502 07:58:52.398111 26995 raft_consensus.cc:482] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:52.398195 26995 raft_consensus.cc:504] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:52.398504 26995 leader_election.cc:288] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:52.398753 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "355e38f2928541cb851924f66cb5550e" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:52.398752 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "355e38f2928541cb851924f66cb5550e" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:52.398855 22845 raft_consensus.cc:2454] T 355e38f2928541cb851924f66cb5550e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:52.398855 22979 raft_consensus.cc:2454] T 355e38f2928541cb851924f66cb5550e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:52.399057 22651 leader_election.cc:302] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:52.399225 26995 raft_consensus.cc:2781] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:52.399264 26995 raft_consensus.cc:482] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:52.399286 26995 raft_consensus.cc:3037] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:52.399919 26995 raft_consensus.cc:504] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:52.400167 26995 leader_election.cc:288] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:52.400377 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "355e38f2928541cb851924f66cb5550e" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:52.400485 22979 raft_consensus.cc:3037] T 355e38f2928541cb851924f66cb5550e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:52.400501 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "355e38f2928541cb851924f66cb5550e" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:52.400578 22845 raft_consensus.cc:3037] T 355e38f2928541cb851924f66cb5550e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:52.400779 30199 tablet.cc:1722] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:52.401003 30199 tablet.cc:1620] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:58:52.401317 22979 raft_consensus.cc:2454] T 355e38f2928541cb851924f66cb5550e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:52.401396 22845 raft_consensus.cc:2454] T 355e38f2928541cb851924f66cb5550e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:52.401512 22651 leader_election.cc:302] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:52.401640 26995 raft_consensus.cc:2781] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:52.401790 26995 raft_consensus.cc:686] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:52.401929 26995 consensus_queue.cc:237] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:52.403612 22442 catalog_manager.cc:5484] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:52.421829 27808 raft_consensus.cc:482] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:52.421918 27808 raft_consensus.cc:504] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:52.422204 27808 leader_election.cc:288] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:52.422530 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5b7fbabf24b240879fe8f15bcfa4b377" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:52.422649 24872 raft_consensus.cc:2454] T 5b7fbabf24b240879fe8f15bcfa4b377 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:52.422885 25199 leader_election.cc:302] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
W20240502 07:58:52.423025 25201 leader_election.cc:334] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:52.423028 27808 raft_consensus.cc:2781] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:52.423105 27808 raft_consensus.cc:482] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:52.423142 27808 raft_consensus.cc:3037] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:52.423816 27808 raft_consensus.cc:504] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:52.424116 27808 leader_election.cc:288] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:52.424422 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5b7fbabf24b240879fe8f15bcfa4b377" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:52.424541 24872 raft_consensus.cc:3037] T 5b7fbabf24b240879fe8f15bcfa4b377 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
W20240502 07:58:52.425020 25201 leader_election.cc:334] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:52.425441 24872 raft_consensus.cc:2454] T 5b7fbabf24b240879fe8f15bcfa4b377 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:52.425662 25199 leader_election.cc:302] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters: 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:52.425825 27808 raft_consensus.cc:2781] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:52.425909 27808 raft_consensus.cc:686] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:52.426031 27808 consensus_queue.cc:237] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:52.427619 22442 catalog_manager.cc:5484] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } }
W20240502 07:58:52.466693 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:58:52.472175 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 07:58:52.490579 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 07:58:52.496201 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:52.500545 30199 tablet.cc:1722] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:52.500854 30199 tablet.cc:1620] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:58:52.501363 30200 tablet.cc:1722] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:52.501603 30200 tablet.cc:1620] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:52.505422 22442 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=0a9339f7788649aea19993a66e360b02] alter complete (version 2)
I20240502 07:58:52.515944 29982 raft_consensus.cc:482] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:52.516033 29982 raft_consensus.cc:504] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:52.516403 29982 leader_election.cc:288] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:52.516695 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f1862e3480644235adaf4a58f2c45bd9" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:52.516819 24872 raft_consensus.cc:2454] T f1862e3480644235adaf4a58f2c45bd9 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:52.517053 25068 leader_election.cc:302] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:52.517195 29982 raft_consensus.cc:2781] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:52.517253 29982 raft_consensus.cc:482] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:52.517287 29982 raft_consensus.cc:3037] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
W20240502 07:58:52.517268 25070 leader_election.cc:334] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:52.518050 29982 raft_consensus.cc:504] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:52.518283 29982 leader_election.cc:288] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857)
I20240502 07:58:52.518555 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f1862e3480644235adaf4a58f2c45bd9" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:52.518653 24872 raft_consensus.cc:3037] T f1862e3480644235adaf4a58f2c45bd9 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
W20240502 07:58:52.519079 25070 leader_election.cc:334] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: RPC error from VoteRequest() call to peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:52.519459 24872 raft_consensus.cc:2454] T f1862e3480644235adaf4a58f2c45bd9 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:52.519686 25068 leader_election.cc:302] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters: 74f9aa91c2c448f980b512fe2e8d672e
I20240502 07:58:52.519809 29982 raft_consensus.cc:2781] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:52.519877 29982 raft_consensus.cc:686] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:58:52.519990 29982 consensus_queue.cc:237] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:52.521484 22444 catalog_manager.cc:5484] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } }
W20240502 07:58:52.535336 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 07:58:52.557936 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 07:58:52.585269 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 07:58:52.588920 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 07:58:52.592983 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 07:58:52.608609 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:52.612848 30199 tablet.cc:1722] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:52.613307 30200 tablet.cc:1722] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:52.617482 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=0a9339f7788649aea19993a66e360b02] alter complete (version 3)
W20240502 07:58:52.656558 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 07:58:52.679991 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 07:58:52.720065 22444 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:52.720187 22444 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:52.722199 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 2a8935d073c74e9a80635b88d9fb0a6f
I20240502 07:58:52.722699 25115 tablet_service.cc:1514] Processing DeleteTablet for tablet 2a8935d073c74e9a80635b88d9fb0a6f with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:52 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:52.722718 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet 2a8935d073c74e9a80635b88d9fb0a6f with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:52 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:52.722986 30268 tablet_replica.cc:330] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:52.723168 30268 raft_consensus.cc:2229] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:52.723315 30268 raft_consensus.cc:2258] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:52.723863 30268 ts_tablet_manager.cc:1875] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:52.724961 30269 tablet_replica.cc:330] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:52.725149 30269 raft_consensus.cc:2229] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:52.725359 30269 raft_consensus.cc:2258] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:52.725906 30269 ts_tablet_manager.cc:1875] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:52.726296 30268 ts_tablet_manager.cc:1888] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:52.726405 30268 log.cc:1196] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/2a8935d073c74e9a80635b88d9fb0a6f
I20240502 07:58:52.726702 30268 ts_tablet_manager.cc:1909] T 2a8935d073c74e9a80635b88d9fb0a6f P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:52.727926 30269 ts_tablet_manager.cc:1888] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:52.728044 30269 log.cc:1196] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/2a8935d073c74e9a80635b88d9fb0a6f
I20240502 07:58:52.728062 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet 2a8935d073c74e9a80635b88d9fb0a6f (table rebalancer_dynamic_table_new_name [id=0a9339f7788649aea19993a66e360b02]) successfully deleted
I20240502 07:58:52.728343 30269 ts_tablet_manager.cc:1909] T 2a8935d073c74e9a80635b88d9fb0a6f P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:52.728998 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 2a8935d073c74e9a80635b88d9fb0a6f (table rebalancer_dynamic_table_new_name [id=0a9339f7788649aea19993a66e360b02]) successfully deleted
I20240502 07:58:52.824347 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:52.828823 24850 tablet_service.cc:1467] Processing CreateTablet for tablet f880e865c56243a586d2404fd7212aa8 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=b918bf5efbdc43de9f9b669e85734437]), partition=
I20240502 07:58:52.828910 25115 tablet_service.cc:1467] Processing CreateTablet for tablet f880e865c56243a586d2404fd7212aa8 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=b918bf5efbdc43de9f9b669e85734437]), partition=
I20240502 07:58:52.829089 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f880e865c56243a586d2404fd7212aa8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:52.829078 25246 tablet_service.cc:1467] Processing CreateTablet for tablet f880e865c56243a586d2404fd7212aa8 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=b918bf5efbdc43de9f9b669e85734437]), partition=
I20240502 07:58:52.829144 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f880e865c56243a586d2404fd7212aa8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:52.829293 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f880e865c56243a586d2404fd7212aa8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:52.831187 30191 tablet_bootstrap.cc:492] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:52.831207 30184 tablet_bootstrap.cc:492] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:52.831988 30184 tablet_bootstrap.cc:654] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:52.831988 30191 tablet_bootstrap.cc:654] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:52.832552 29973 tablet_bootstrap.cc:492] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:52.832986 30191 tablet_bootstrap.cc:492] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:52.833048 30191 ts_tablet_manager.cc:1367] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:52.833513 29973 tablet_bootstrap.cc:654] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:52.833480 30191 raft_consensus.cc:348] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:52.833563 30191 raft_consensus.cc:374] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:52.833583 30191 raft_consensus.cc:729] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:52.833675 30191 consensus_queue.cc:260] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:52.833925 30191 ts_tablet_manager.cc:1398] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:52.834497 30184 tablet_bootstrap.cc:492] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:52.834582 30184 ts_tablet_manager.cc:1367] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:52.835075 30184 raft_consensus.cc:348] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:52.835144 30184 raft_consensus.cc:374] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:52.835163 30184 raft_consensus.cc:729] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:52.835237 30184 consensus_queue.cc:260] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:52.835427 30184 ts_tablet_manager.cc:1398] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:52.835856 29973 tablet_bootstrap.cc:492] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:52.835942 29973 ts_tablet_manager.cc:1367] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:52.836304 29973 raft_consensus.cc:348] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:52.836402 29973 raft_consensus.cc:374] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:52.836441 29973 raft_consensus.cc:729] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:52.836545 29973 consensus_queue.cc:260] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:52.836814 29973 ts_tablet_manager.cc:1398] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
W20240502 07:58:52.859387 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:58:52.877990 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 07:58:52.905088 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 07:58:52.919485 27808 consensus_queue.cc:1035] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:52.923803 29982 consensus_queue.cc:1035] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:58:52.927232 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 07:58:52.939637 26995 consensus_queue.cc:1035] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:52.943459 30312 consensus_queue.cc:1035] T 355e38f2928541cb851924f66cb5550e P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:52.984748 29941 raft_consensus.cc:482] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:52.984849 29941 raft_consensus.cc:504] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:52.985220 29941 leader_election.cc:288] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:52.985476 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f880e865c56243a586d2404fd7212aa8" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:52.985497 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f880e865c56243a586d2404fd7212aa8" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:52.985616 25133 raft_consensus.cc:2454] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:52.985616 24870 raft_consensus.cc:2454] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:52.985831 25202 leader_election.cc:302] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:52.985980 29941 raft_consensus.cc:2781] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:52.986035 29941 raft_consensus.cc:482] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:52.986063 29941 raft_consensus.cc:3037] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:52.986768 29941 raft_consensus.cc:504] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:52.987021 29941 leader_election.cc:288] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:52.987280 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f880e865c56243a586d2404fd7212aa8" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:52.987280 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f880e865c56243a586d2404fd7212aa8" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:52.987380 24870 raft_consensus.cc:3037] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:52.987380 25133 raft_consensus.cc:3037] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:52.988251 24870 raft_consensus.cc:2454] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:52.988251 25133 raft_consensus.cc:2454] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:52.988533 25199 leader_election.cc:302] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:52.988699 29941 raft_consensus.cc:2781] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:52.988775 29941 raft_consensus.cc:686] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:52.988893 29941 consensus_queue.cc:237] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:52.990568 22442 catalog_manager.cc:5484] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } }
W20240502 07:58:53.022804 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): CreateTablet RPC failed for tablet 5b7fbabf24b240879fe8f15bcfa4b377: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:53.044345 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_28"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:53.052423 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 1a2ea813c30841f7b3ad4f8f77ce168d (DEFAULT_TABLE table=rebalancer_extra_table_28 [id=90d6214b31ef48a6aa31254e23abe35c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.052548 25115 tablet_service.cc:1467] Processing CreateTablet for tablet ee908769faeb4a4489ea9d2b5cc7fec6 (DEFAULT_TABLE table=rebalancer_extra_table_28 [id=90d6214b31ef48a6aa31254e23abe35c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.052721 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1a2ea813c30841f7b3ad4f8f77ce168d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.052835 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ee908769faeb4a4489ea9d2b5cc7fec6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.052049 24850 tablet_service.cc:1467] Processing CreateTablet for tablet ee908769faeb4a4489ea9d2b5cc7fec6 (DEFAULT_TABLE table=rebalancer_extra_table_28 [id=90d6214b31ef48a6aa31254e23abe35c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.054041 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ee908769faeb4a4489ea9d2b5cc7fec6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.054208 22697 tablet_service.cc:1467] Processing CreateTablet for tablet ee908769faeb4a4489ea9d2b5cc7fec6 (DEFAULT_TABLE table=rebalancer_extra_table_28 [id=90d6214b31ef48a6aa31254e23abe35c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.054455 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ee908769faeb4a4489ea9d2b5cc7fec6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.055076 29973 tablet_bootstrap.cc:492] T ee908769faeb4a4489ea9d2b5cc7fec6 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:53.055985 29973 tablet_bootstrap.cc:654] T ee908769faeb4a4489ea9d2b5cc7fec6 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.056707 30317 tablet_bootstrap.cc:492] T ee908769faeb4a4489ea9d2b5cc7fec6 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:53.057099 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 51989bc9da53484d86dba60b0d6c3ff5 (DEFAULT_TABLE table=rebalancer_extra_table_28 [id=90d6214b31ef48a6aa31254e23abe35c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.057132 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 51989bc9da53484d86dba60b0d6c3ff5 (DEFAULT_TABLE table=rebalancer_extra_table_28 [id=90d6214b31ef48a6aa31254e23abe35c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.057333 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 51989bc9da53484d86dba60b0d6c3ff5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.057341 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 51989bc9da53484d86dba60b0d6c3ff5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.057709 30317 tablet_bootstrap.cc:654] T ee908769faeb4a4489ea9d2b5cc7fec6 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.058545 30317 tablet_bootstrap.cc:492] T ee908769faeb4a4489ea9d2b5cc7fec6 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:53.058600 30317 ts_tablet_manager.cc:1367] T ee908769faeb4a4489ea9d2b5cc7fec6 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:53.058887 30317 raft_consensus.cc:348] T ee908769faeb4a4489ea9d2b5cc7fec6 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:53.058960 30317 raft_consensus.cc:374] T ee908769faeb4a4489ea9d2b5cc7fec6 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.058979 30317 raft_consensus.cc:729] T ee908769faeb4a4489ea9d2b5cc7fec6 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.059077 30317 consensus_queue.cc:260] T ee908769faeb4a4489ea9d2b5cc7fec6 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:53.059247 30191 tablet_bootstrap.cc:492] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:53.059340 30317 ts_tablet_manager.cc:1398] T ee908769faeb4a4489ea9d2b5cc7fec6 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:53.059531 30320 tablet_bootstrap.cc:492] T 51989bc9da53484d86dba60b0d6c3ff5 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:53.059906 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 1a2ea813c30841f7b3ad4f8f77ce168d (DEFAULT_TABLE table=rebalancer_extra_table_28 [id=90d6214b31ef48a6aa31254e23abe35c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.060058 30191 tablet_bootstrap.cc:654] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.060010 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 51989bc9da53484d86dba60b0d6c3ff5 (DEFAULT_TABLE table=rebalancer_extra_table_28 [id=90d6214b31ef48a6aa31254e23abe35c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.060179 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1a2ea813c30841f7b3ad4f8f77ce168d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.060266 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 51989bc9da53484d86dba60b0d6c3ff5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.060513 30320 tablet_bootstrap.cc:654] T 51989bc9da53484d86dba60b0d6c3ff5 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.060808 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 1a2ea813c30841f7b3ad4f8f77ce168d (DEFAULT_TABLE table=rebalancer_extra_table_28 [id=90d6214b31ef48a6aa31254e23abe35c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.060992 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1a2ea813c30841f7b3ad4f8f77ce168d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.062614 30320 tablet_bootstrap.cc:492] T 51989bc9da53484d86dba60b0d6c3ff5 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:53.062698 30320 ts_tablet_manager.cc:1367] T 51989bc9da53484d86dba60b0d6c3ff5 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:53.056638 30184 tablet_bootstrap.cc:492] T 1a2ea813c30841f7b3ad4f8f77ce168d P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:53.063230 30320 raft_consensus.cc:348] T 51989bc9da53484d86dba60b0d6c3ff5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:53.063324 30320 raft_consensus.cc:374] T 51989bc9da53484d86dba60b0d6c3ff5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.063354 30320 raft_consensus.cc:729] T 51989bc9da53484d86dba60b0d6c3ff5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.063449 30320 consensus_queue.cc:260] T 51989bc9da53484d86dba60b0d6c3ff5 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:53.063758 30320 ts_tablet_manager.cc:1398] T 51989bc9da53484d86dba60b0d6c3ff5 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:53.063977 30184 tablet_bootstrap.cc:654] T 1a2ea813c30841f7b3ad4f8f77ce168d P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.065966 30184 tablet_bootstrap.cc:492] T 1a2ea813c30841f7b3ad4f8f77ce168d P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:53.066052 30184 ts_tablet_manager.cc:1367] T 1a2ea813c30841f7b3ad4f8f77ce168d P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.009s user 0.002s sys 0.000s
I20240502 07:58:53.066475 30191 tablet_bootstrap.cc:492] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:53.062026 30323 tablet_bootstrap.cc:492] T 1a2ea813c30841f7b3ad4f8f77ce168d P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:53.066484 30184 raft_consensus.cc:348] T 1a2ea813c30841f7b3ad4f8f77ce168d P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:53.066550 30191 ts_tablet_manager.cc:1367] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 07:58:53.066570 30184 raft_consensus.cc:374] T 1a2ea813c30841f7b3ad4f8f77ce168d P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.066601 30184 raft_consensus.cc:729] T 1a2ea813c30841f7b3ad4f8f77ce168d P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.066720 30184 consensus_queue.cc:260] T 1a2ea813c30841f7b3ad4f8f77ce168d P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:53.066974 30184 ts_tablet_manager.cc:1398] T 1a2ea813c30841f7b3ad4f8f77ce168d P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:53.066977 30191 raft_consensus.cc:348] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:53.067075 30191 raft_consensus.cc:374] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.067107 30191 raft_consensus.cc:729] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.067153 30184 tablet_bootstrap.cc:492] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:53.067216 30191 consensus_queue.cc:260] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:53.067464 30191 ts_tablet_manager.cc:1398] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:53.067935 30184 tablet_bootstrap.cc:654] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.068805 30324 tablet_bootstrap.cc:492] T 51989bc9da53484d86dba60b0d6c3ff5 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:53.069932 30184 tablet_bootstrap.cc:492] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:53.070008 30184 ts_tablet_manager.cc:1367] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:53.070086 30324 tablet_bootstrap.cc:654] T 51989bc9da53484d86dba60b0d6c3ff5 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.070430 30184 raft_consensus.cc:348] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:53.070516 30184 raft_consensus.cc:374] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.070550 30184 raft_consensus.cc:729] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.070667 30184 consensus_queue.cc:260] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:53.070842 30323 tablet_bootstrap.cc:654] T 1a2ea813c30841f7b3ad4f8f77ce168d P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.070931 29973 tablet_bootstrap.cc:492] T ee908769faeb4a4489ea9d2b5cc7fec6 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:53.070961 30184 ts_tablet_manager.cc:1398] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:53.071028 29973 ts_tablet_manager.cc:1367] T ee908769faeb4a4489ea9d2b5cc7fec6 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.016s user 0.002s sys 0.000s
I20240502 07:58:53.071446 29973 raft_consensus.cc:348] T ee908769faeb4a4489ea9d2b5cc7fec6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:53.071585 29973 raft_consensus.cc:374] T ee908769faeb4a4489ea9d2b5cc7fec6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.071616 29973 raft_consensus.cc:729] T ee908769faeb4a4489ea9d2b5cc7fec6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.071719 29973 consensus_queue.cc:260] T ee908769faeb4a4489ea9d2b5cc7fec6 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:53.071928 30323 tablet_bootstrap.cc:492] T 1a2ea813c30841f7b3ad4f8f77ce168d P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:53.072000 30323 ts_tablet_manager.cc:1367] T 1a2ea813c30841f7b3ad4f8f77ce168d P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.010s user 0.001s sys 0.000s
I20240502 07:58:53.071991 29973 ts_tablet_manager.cc:1398] T ee908769faeb4a4489ea9d2b5cc7fec6 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:53.072149 29973 tablet_bootstrap.cc:492] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:53.072329 30323 raft_consensus.cc:348] T 1a2ea813c30841f7b3ad4f8f77ce168d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:53.072429 30323 raft_consensus.cc:374] T 1a2ea813c30841f7b3ad4f8f77ce168d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.072464 30323 raft_consensus.cc:729] T 1a2ea813c30841f7b3ad4f8f77ce168d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.072572 30323 consensus_queue.cc:260] T 1a2ea813c30841f7b3ad4f8f77ce168d P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:53.072863 30323 ts_tablet_manager.cc:1398] T 1a2ea813c30841f7b3ad4f8f77ce168d P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:53.073081 29973 tablet_bootstrap.cc:654] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.073917 30324 tablet_bootstrap.cc:492] T 51989bc9da53484d86dba60b0d6c3ff5 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:53.074049 30324 ts_tablet_manager.cc:1367] T 51989bc9da53484d86dba60b0d6c3ff5 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:58:53.074534 30324 raft_consensus.cc:348] T 51989bc9da53484d86dba60b0d6c3ff5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:53.074715 30324 raft_consensus.cc:374] T 51989bc9da53484d86dba60b0d6c3ff5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.074770 30324 raft_consensus.cc:729] T 51989bc9da53484d86dba60b0d6c3ff5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.074959 30324 consensus_queue.cc:260] T 51989bc9da53484d86dba60b0d6c3ff5 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:53.075278 30324 ts_tablet_manager.cc:1398] T 51989bc9da53484d86dba60b0d6c3ff5 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:53.077325 29973 tablet_bootstrap.cc:492] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:53.077402 29973 ts_tablet_manager.cc:1367] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:53.077845 29973 raft_consensus.cc:348] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:53.077946 29973 raft_consensus.cc:374] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.077980 29973 raft_consensus.cc:729] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.078068 29973 consensus_queue.cc:260] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:53.078297 29973 ts_tablet_manager.cc:1398] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
W20240502 07:58:53.107437 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 07:58:53.121615 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:53.125082 24870 raft_consensus.cc:1261] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:53.125083 25133 raft_consensus.cc:1261] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:53.125466 27808 consensus_queue.cc:1035] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:53.125617 29941 consensus_queue.cc:1035] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:53.127913 30332 tablet.cc:1722] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:53.128204 30332 tablet.cc:1620] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:53.128369 22440 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet f880e865c56243a586d2404fd7212aa8 (table rebalancer_dynamic_table [id=b918bf5efbdc43de9f9b669e85734437]). Expected version 1 got 0
I20240502 07:58:53.130079 30333 tablet.cc:1722] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:53.130393 30333 tablet.cc:1620] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:53.131039 30334 tablet.cc:1722] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:53.131412 30334 tablet.cc:1620] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:53.131414 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=b918bf5efbdc43de9f9b669e85734437] alter complete (version 1)
I20240502 07:58:53.132376 22444 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet f880e865c56243a586d2404fd7212aa8 (table rebalancer_dynamic_table [id=b918bf5efbdc43de9f9b669e85734437]). Expected version 1 got 0
I20240502 07:58:53.133198 22440 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet f880e865c56243a586d2404fd7212aa8 (table rebalancer_dynamic_table [id=b918bf5efbdc43de9f9b669e85734437]). Expected version 1 got 0
I20240502 07:58:53.164255 30293 raft_consensus.cc:482] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:53.164359 30293 raft_consensus.cc:504] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:53.164691 30293 leader_election.cc:288] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:53.164973 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a2ea813c30841f7b3ad4f8f77ce168d" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:53.165124 24870 raft_consensus.cc:2454] T 1a2ea813c30841f7b3ad4f8f77ce168d P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:53.165171 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a2ea813c30841f7b3ad4f8f77ce168d" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:53.165303 22586 raft_consensus.cc:2454] T 1a2ea813c30841f7b3ad4f8f77ce168d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:53.165463 25068 leader_election.cc:302] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:53.165627 30293 raft_consensus.cc:2781] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:53.165674 30293 raft_consensus.cc:482] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:53.165721 30293 raft_consensus.cc:3037] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:53.166635 30293 raft_consensus.cc:504] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:53.166903 30293 leader_election.cc:288] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:53.167160 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a2ea813c30841f7b3ad4f8f77ce168d" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:53.167215 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1a2ea813c30841f7b3ad4f8f77ce168d" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:53.167273 24870 raft_consensus.cc:3037] T 1a2ea813c30841f7b3ad4f8f77ce168d P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:53.167305 22586 raft_consensus.cc:3037] T 1a2ea813c30841f7b3ad4f8f77ce168d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:53.168190 22586 raft_consensus.cc:2454] T 1a2ea813c30841f7b3ad4f8f77ce168d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:53.168228 24870 raft_consensus.cc:2454] T 1a2ea813c30841f7b3ad4f8f77ce168d P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:53.168445 25071 leader_election.cc:302] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:53.168618 30293 raft_consensus.cc:2781] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:53.168710 30293 raft_consensus.cc:686] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:58:53.168834 30293 consensus_queue.cc:237] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:53.170668 22442 catalog_manager.cc:5484] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:53.236085 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:53.240396 30332 tablet.cc:1722] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:53.240731 30332 tablet.cc:1620] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:53.241137 30334 tablet.cc:1722] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:53.241382 30334 tablet.cc:1620] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:53.241873 30333 tablet.cc:1722] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:53.242134 30333 tablet.cc:1620] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:53.245642 29941 raft_consensus.cc:482] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:53.245736 29941 raft_consensus.cc:504] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:53.245667 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=b918bf5efbdc43de9f9b669e85734437] alter complete (version 2)
I20240502 07:58:53.246083 29941 leader_election.cc:288] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:53.246436 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "51989bc9da53484d86dba60b0d6c3ff5" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:53.246565 22979 raft_consensus.cc:2454] T 51989bc9da53484d86dba60b0d6c3ff5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:53.246812 25200 leader_election.cc:302] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:53.246944 29941 raft_consensus.cc:2781] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:53.246994 29941 raft_consensus.cc:482] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:53.247021 29941 raft_consensus.cc:3037] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:53.247640 29941 raft_consensus.cc:504] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:53.247931 29941 leader_election.cc:288] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:53.248103 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "51989bc9da53484d86dba60b0d6c3ff5" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:53.248194 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "51989bc9da53484d86dba60b0d6c3ff5" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:53.248338 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "51989bc9da53484d86dba60b0d6c3ff5" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:53.248422 22979 raft_consensus.cc:3037] T 51989bc9da53484d86dba60b0d6c3ff5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:53.249137 22979 raft_consensus.cc:2454] T 51989bc9da53484d86dba60b0d6c3ff5 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:53.248229 22847 raft_consensus.cc:2454] T 51989bc9da53484d86dba60b0d6c3ff5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:53.249387 25200 leader_election.cc:302] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7eced220cf1642f3873fc385b6191e69; no voters: 7db4178eed174225bfb24f5f9445ce88
I20240502 07:58:53.249547 29941 raft_consensus.cc:2781] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:53.249617 29941 raft_consensus.cc:686] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:53.249724 29941 consensus_queue.cc:237] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:53.251410 22440 catalog_manager.cc:5484] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:53.282238 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 07:58:53.346743 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:53.350910 30332 tablet.cc:1722] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:53.351869 30333 tablet.cc:1722] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:53.353116 30334 tablet.cc:1722] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:53.355756 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=b918bf5efbdc43de9f9b669e85734437] alter complete (version 3)
I20240502 07:58:53.390615 30195 raft_consensus.cc:482] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:53.390710 30195 raft_consensus.cc:504] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:53.391037 30195 leader_election.cc:288] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:53.391278 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ee908769faeb4a4489ea9d2b5cc7fec6" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:53.391357 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ee908769faeb4a4489ea9d2b5cc7fec6" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:53.391405 25133 raft_consensus.cc:2454] T ee908769faeb4a4489ea9d2b5cc7fec6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:53.391455 22716 raft_consensus.cc:2454] T ee908769faeb4a4489ea9d2b5cc7fec6 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:53.391647 24808 leader_election.cc:302] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:53.391897 30195 raft_consensus.cc:2781] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:53.391961 30195 raft_consensus.cc:482] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:53.391999 30195 raft_consensus.cc:3037] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:53.392812 30195 raft_consensus.cc:504] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:53.393144 30195 leader_election.cc:288] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:53.393494 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ee908769faeb4a4489ea9d2b5cc7fec6" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:53.393581 22716 raft_consensus.cc:3037] T ee908769faeb4a4489ea9d2b5cc7fec6 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:53.393559 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ee908769faeb4a4489ea9d2b5cc7fec6" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:53.393656 25133 raft_consensus.cc:3037] T ee908769faeb4a4489ea9d2b5cc7fec6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:53.394299 22716 raft_consensus.cc:2454] T ee908769faeb4a4489ea9d2b5cc7fec6 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:53.394438 25133 raft_consensus.cc:2454] T ee908769faeb4a4489ea9d2b5cc7fec6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:53.394661 24808 leader_election.cc:302] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:53.394829 30195 raft_consensus.cc:2781] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:53.394932 30195 raft_consensus.cc:686] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:58:53.395072 30195 consensus_queue.cc:237] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:53.396929 22436 catalog_manager.cc:5484] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:53.405699 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 07:58:53.425606 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:53.459265 22436 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:53.459385 22436 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:53.461370 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet f880e865c56243a586d2404fd7212aa8
I20240502 07:58:53.461871 24850 tablet_service.cc:1514] Processing DeleteTablet for tablet f880e865c56243a586d2404fd7212aa8 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:53 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:53.461894 25115 tablet_service.cc:1514] Processing DeleteTablet for tablet f880e865c56243a586d2404fd7212aa8 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:53 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:53.462143 30398 tablet_replica.cc:330] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:53.462206 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet f880e865c56243a586d2404fd7212aa8 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:53 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:53.462239 30399 tablet_replica.cc:330] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:53.462283 30398 raft_consensus.cc:2229] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:53.462366 30399 raft_consensus.cc:2229] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:53.462410 30398 raft_consensus.cc:2258] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:53.462455 30400 tablet_replica.cc:330] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:53.462478 30399 raft_consensus.cc:2258] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:53.462587 30400 raft_consensus.cc:2229] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:53.462826 30400 raft_consensus.cc:2258] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:53.463101 30399 ts_tablet_manager.cc:1875] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:53.463188 30398 ts_tablet_manager.cc:1875] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:53.463284 30400 ts_tablet_manager.cc:1875] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:53.465500 30399 ts_tablet_manager.cc:1888] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:53.465497 30398 ts_tablet_manager.cc:1888] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:53.465564 30399 log.cc:1196] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/f880e865c56243a586d2404fd7212aa8
I20240502 07:58:53.465571 30398 log.cc:1196] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/f880e865c56243a586d2404fd7212aa8
I20240502 07:58:53.465605 30400 ts_tablet_manager.cc:1888] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:53.465701 30400 log.cc:1196] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/f880e865c56243a586d2404fd7212aa8
I20240502 07:58:53.465839 30399 ts_tablet_manager.cc:1909] T f880e865c56243a586d2404fd7212aa8 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:53.465873 30398 ts_tablet_manager.cc:1909] T f880e865c56243a586d2404fd7212aa8 P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:53.465941 30400 ts_tablet_manager.cc:1909] T f880e865c56243a586d2404fd7212aa8 P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:58:53.466361 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet f880e865c56243a586d2404fd7212aa8 (table rebalancer_dynamic_table_new_name [id=b918bf5efbdc43de9f9b669e85734437]) successfully deleted
I20240502 07:58:53.466431 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet f880e865c56243a586d2404fd7212aa8 (table rebalancer_dynamic_table_new_name [id=b918bf5efbdc43de9f9b669e85734437]) successfully deleted
I20240502 07:58:53.466567 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet f880e865c56243a586d2404fd7212aa8 (table rebalancer_dynamic_table_new_name [id=b918bf5efbdc43de9f9b669e85734437]) successfully deleted
W20240502 07:58:53.509137 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 07:58:53.553324 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 07:58:53.562812 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:53.567651 25115 tablet_service.cc:1467] Processing CreateTablet for tablet a46d64c31804473593e0c258e9d807cf (DEFAULT_TABLE table=rebalancer_dynamic_table [id=baf1c515beac468790924980777b15ad]), partition=
I20240502 07:58:53.567744 24850 tablet_service.cc:1467] Processing CreateTablet for tablet a46d64c31804473593e0c258e9d807cf (DEFAULT_TABLE table=rebalancer_dynamic_table [id=baf1c515beac468790924980777b15ad]), partition=
I20240502 07:58:53.567899 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a46d64c31804473593e0c258e9d807cf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.567953 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a46d64c31804473593e0c258e9d807cf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.569679 29973 tablet_bootstrap.cc:492] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:53.569821 30184 tablet_bootstrap.cc:492] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:53.570398 29973 tablet_bootstrap.cc:654] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.570539 30184 tablet_bootstrap.cc:654] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.571403 22827 tablet_service.cc:1467] Processing CreateTablet for tablet a46d64c31804473593e0c258e9d807cf (DEFAULT_TABLE table=rebalancer_dynamic_table [id=baf1c515beac468790924980777b15ad]), partition=
I20240502 07:58:53.571509 29973 tablet_bootstrap.cc:492] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:53.571576 29973 ts_tablet_manager.cc:1367] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:53.571609 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a46d64c31804473593e0c258e9d807cf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.571998 29973 raft_consensus.cc:348] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:53.572090 29973 raft_consensus.cc:374] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.572124 29973 raft_consensus.cc:729] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.572224 29973 consensus_queue.cc:260] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:53.572472 29973 ts_tablet_manager.cc:1398] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:53.573503 30324 tablet_bootstrap.cc:492] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:53.573541 30184 tablet_bootstrap.cc:492] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:53.573606 30184 ts_tablet_manager.cc:1367] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:53.573982 30184 raft_consensus.cc:348] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:53.574066 30184 raft_consensus.cc:374] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.574092 30184 raft_consensus.cc:729] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.574183 30184 consensus_queue.cc:260] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:53.574395 30184 ts_tablet_manager.cc:1398] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:53.574393 30324 tablet_bootstrap.cc:654] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.575860 30324 tablet_bootstrap.cc:492] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:53.575940 30324 ts_tablet_manager.cc:1367] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:53.576344 30324 raft_consensus.cc:348] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:53.576434 30324 raft_consensus.cc:374] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.576465 30324 raft_consensus.cc:729] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.576567 30324 consensus_queue.cc:260] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:53.576902 30324 ts_tablet_manager.cc:1398] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
W20240502 07:58:53.625180 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 07:58:53.690132 30195 raft_consensus.cc:482] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:53.690258 30195 raft_consensus.cc:504] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:53.690562 30195 leader_election.cc:288] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:53.690829 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a46d64c31804473593e0c258e9d807cf" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:53.690987 25133 raft_consensus.cc:2454] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:53.690984 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a46d64c31804473593e0c258e9d807cf" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:53.691092 22847 raft_consensus.cc:2454] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:53.691279 24808 leader_election.cc:302] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:53.691478 30195 raft_consensus.cc:2781] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:53.691545 30195 raft_consensus.cc:482] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:53.691591 30195 raft_consensus.cc:3037] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:53.692706 30195 raft_consensus.cc:504] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:53.693010 30195 leader_election.cc:288] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:58:53.693302 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a46d64c31804473593e0c258e9d807cf" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:53.693302 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a46d64c31804473593e0c258e9d807cf" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:53.693395 25133 raft_consensus.cc:3037] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:53.693395 22847 raft_consensus.cc:3037] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:53.694294 22847 raft_consensus.cc:2454] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:53.694294 25133 raft_consensus.cc:2454] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:53.694546 24808 leader_election.cc:302] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:53.694823 30195 raft_consensus.cc:2781] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:53.694923 30195 raft_consensus.cc:686] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
W20240502 07:58:53.695082 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 07:58:53.695070 30195 consensus_queue.cc:237] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:58:53.696853 22440 catalog_manager.cc:5484] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:53.732803 30293 consensus_queue.cc:1035] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:53.736873 30438 consensus_queue.cc:1035] T 1a2ea813c30841f7b3ad4f8f77ce168d P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20240502 07:58:53.823045 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:53.827018 25133 raft_consensus.cc:1261] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:53.827204 22847 raft_consensus.cc:1261] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:53.827721 30441 consensus_queue.cc:1035] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:53.827894 30270 consensus_queue.cc:1035] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:53.830829 22436 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet a46d64c31804473593e0c258e9d807cf (table rebalancer_dynamic_table [id=baf1c515beac468790924980777b15ad]). Expected version 1 got 0
I20240502 07:58:53.831277 22444 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet a46d64c31804473593e0c258e9d807cf (table rebalancer_dynamic_table [id=baf1c515beac468790924980777b15ad]). Expected version 1 got 0
I20240502 07:58:53.832008 30333 tablet.cc:1722] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:53.832372 30333 tablet.cc:1620] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:53.832947 30334 tablet.cc:1722] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:53.833216 30334 tablet.cc:1620] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:53.834273 30445 tablet.cc:1722] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:53.834712 30445 tablet.cc:1620] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:58:53.836948 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=baf1c515beac468790924980777b15ad] alter complete (version 1)
I20240502 07:58:53.844244 29941 consensus_queue.cc:1035] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:53.845137 30270 consensus_queue.cc:1035] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:53.847808 22847 raft_consensus.cc:3037] T 51989bc9da53484d86dba60b0d6c3ff5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:53.849416 27495 consensus_queue.cc:1035] T 51989bc9da53484d86dba60b0d6c3ff5 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:53.851361 30270 consensus_queue.cc:1035] T ee908769faeb4a4489ea9d2b5cc7fec6 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:53.923389 22440 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_30"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:53.928524 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 5d77d4781250465a88338bb4c303918c (DEFAULT_TABLE table=rebalancer_extra_table_30 [id=9526d64861904fffa42599ed3c96c273]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.928856 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5d77d4781250465a88338bb4c303918c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.928943 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 5d77d4781250465a88338bb4c303918c (DEFAULT_TABLE table=rebalancer_extra_table_30 [id=9526d64861904fffa42599ed3c96c273]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.928984 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 057f08ea1ed34b2fb2ca2b56f87b6b3f (DEFAULT_TABLE table=rebalancer_extra_table_30 [id=9526d64861904fffa42599ed3c96c273]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.929205 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5d77d4781250465a88338bb4c303918c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.929205 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 057f08ea1ed34b2fb2ca2b56f87b6b3f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.931118 30484 tablet_bootstrap.cc:492] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:53.931200 30184 tablet_bootstrap.cc:492] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:53.931354 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 057f08ea1ed34b2fb2ca2b56f87b6b3f (DEFAULT_TABLE table=rebalancer_extra_table_30 [id=9526d64861904fffa42599ed3c96c273]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.931594 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 057f08ea1ed34b2fb2ca2b56f87b6b3f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.931742 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 057f08ea1ed34b2fb2ca2b56f87b6b3f (DEFAULT_TABLE table=rebalancer_extra_table_30 [id=9526d64861904fffa42599ed3c96c273]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.932000 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 057f08ea1ed34b2fb2ca2b56f87b6b3f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.932057 30184 tablet_bootstrap.cc:654] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.932130 30484 tablet_bootstrap.cc:654] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.932693 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 5d77d4781250465a88338bb4c303918c (DEFAULT_TABLE table=rebalancer_extra_table_30 [id=9526d64861904fffa42599ed3c96c273]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.932883 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5d77d4781250465a88338bb4c303918c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.933341 30483 tablet_bootstrap.cc:492] T 5d77d4781250465a88338bb4c303918c P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:53.934258 30483 tablet_bootstrap.cc:654] T 5d77d4781250465a88338bb4c303918c P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.934898 30487 tablet_bootstrap.cc:492] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:53.935214 29973 tablet_bootstrap.cc:492] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:53.935218 30184 tablet_bootstrap.cc:492] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:53.935286 30184 ts_tablet_manager.cc:1367] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:53.935724 30184 raft_consensus.cc:348] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:53.935810 30487 tablet_bootstrap.cc:654] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.935825 30184 raft_consensus.cc:374] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.935853 30184 raft_consensus.cc:729] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.935973 30184 consensus_queue.cc:260] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:53.936112 29973 tablet_bootstrap.cc:654] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.936269 30184 ts_tablet_manager.cc:1398] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:53.936944 30484 tablet_bootstrap.cc:492] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:53.936949 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 4e349c1038774c0c8f8fcebd1dc3f0c6 (DEFAULT_TABLE table=rebalancer_extra_table_30 [id=9526d64861904fffa42599ed3c96c273]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.937016 30484 ts_tablet_manager.cc:1367] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:58:53.937179 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4e349c1038774c0c8f8fcebd1dc3f0c6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.937193 29973 tablet_bootstrap.cc:492] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:53.937256 29973 ts_tablet_manager.cc:1367] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.001s
I20240502 07:58:53.937420 30484 raft_consensus.cc:348] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:53.937502 30484 raft_consensus.cc:374] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.937525 30484 raft_consensus.cc:729] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.937577 29973 raft_consensus.cc:348] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:53.937633 30484 consensus_queue.cc:260] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:53.937670 29973 raft_consensus.cc:374] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.937698 29973 raft_consensus.cc:729] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.937794 29973 consensus_queue.cc:260] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:53.937844 30484 ts_tablet_manager.cc:1398] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:53.938052 29973 ts_tablet_manager.cc:1398] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
W20240502 07:58:53.939827 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:53.940773 30487 tablet_bootstrap.cc:492] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:53.940954 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:53.940846 30487 ts_tablet_manager.cc:1367] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:58:53.942956 30487 raft_consensus.cc:348] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:53.943156 30487 raft_consensus.cc:374] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.943219 30487 raft_consensus.cc:729] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.943378 30487 consensus_queue.cc:260] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:53.943701 30487 ts_tablet_manager.cc:1398] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:53.943780 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 4e349c1038774c0c8f8fcebd1dc3f0c6 (DEFAULT_TABLE table=rebalancer_extra_table_30 [id=9526d64861904fffa42599ed3c96c273]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.944003 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4e349c1038774c0c8f8fcebd1dc3f0c6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.944013 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 4e349c1038774c0c8f8fcebd1dc3f0c6 (DEFAULT_TABLE table=rebalancer_extra_table_30 [id=9526d64861904fffa42599ed3c96c273]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:53.944228 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4e349c1038774c0c8f8fcebd1dc3f0c6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:53.944675 30483 tablet_bootstrap.cc:492] T 5d77d4781250465a88338bb4c303918c P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:53.944753 30483 ts_tablet_manager.cc:1367] T 5d77d4781250465a88338bb4c303918c P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.000s
I20240502 07:58:53.945134 30483 raft_consensus.cc:348] T 5d77d4781250465a88338bb4c303918c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:53.945202 30483 raft_consensus.cc:374] T 5d77d4781250465a88338bb4c303918c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.945219 30483 raft_consensus.cc:729] T 5d77d4781250465a88338bb4c303918c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.945307 30483 consensus_queue.cc:260] T 5d77d4781250465a88338bb4c303918c P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:53.945513 30483 ts_tablet_manager.cc:1398] T 5d77d4781250465a88338bb4c303918c P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:53.945848 29973 tablet_bootstrap.cc:492] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:53.946632 29973 tablet_bootstrap.cc:654] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.947114 30184 tablet_bootstrap.cc:492] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:53.947907 30184 tablet_bootstrap.cc:654] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.949357 30184 tablet_bootstrap.cc:492] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:53.949430 30184 ts_tablet_manager.cc:1367] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:53.949476 30333 tablet.cc:1722] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:53.949694 30333 tablet.cc:1620] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:53.949800 30184 raft_consensus.cc:348] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:53.949883 30184 raft_consensus.cc:374] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.949914 30184 raft_consensus.cc:729] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.950013 30184 consensus_queue.cc:260] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:53.950279 30184 ts_tablet_manager.cc:1398] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:53.952015 30487 tablet_bootstrap.cc:492] T 5d77d4781250465a88338bb4c303918c P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:53.951999 30334 tablet.cc:1722] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:53.952242 30334 tablet.cc:1620] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:53.952486 30445 tablet.cc:1722] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:53.952759 30445 tablet.cc:1620] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:58:53.956441 30487 tablet_bootstrap.cc:654] T 5d77d4781250465a88338bb4c303918c P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.957513 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=baf1c515beac468790924980777b15ad] alter complete (version 2)
I20240502 07:58:53.957720 30491 tablet_bootstrap.cc:492] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:53.958726 30487 tablet_bootstrap.cc:492] T 5d77d4781250465a88338bb4c303918c P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:53.958797 30487 ts_tablet_manager.cc:1367] T 5d77d4781250465a88338bb4c303918c P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.001s
I20240502 07:58:53.958822 30491 tablet_bootstrap.cc:654] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:53.959124 30487 raft_consensus.cc:348] T 5d77d4781250465a88338bb4c303918c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:53.959200 30487 raft_consensus.cc:374] T 5d77d4781250465a88338bb4c303918c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.959231 30487 raft_consensus.cc:729] T 5d77d4781250465a88338bb4c303918c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.959328 30487 consensus_queue.cc:260] T 5d77d4781250465a88338bb4c303918c P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:53.959571 30487 ts_tablet_manager.cc:1398] T 5d77d4781250465a88338bb4c303918c P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:53.959782 29973 tablet_bootstrap.cc:492] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:53.959851 29973 ts_tablet_manager.cc:1367] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.014s user 0.000s sys 0.001s
W20240502 07:58:53.959944 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 07:58:53.960297 30491 tablet_bootstrap.cc:492] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:53.960245 29973 raft_consensus.cc:348] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:53.960335 29973 raft_consensus.cc:374] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.960381 29973 raft_consensus.cc:729] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.960458 30491 ts_tablet_manager.cc:1367] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:53.960517 29973 consensus_queue.cc:260] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:53.960922 29973 ts_tablet_manager.cc:1398] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:53.960942 30491 raft_consensus.cc:348] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:53.961063 30491 raft_consensus.cc:374] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:53.961107 30491 raft_consensus.cc:729] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:53.961246 30491 consensus_queue.cc:260] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:53.961540 30491 ts_tablet_manager.cc:1398] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
W20240502 07:58:53.989444 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 07:58:54.002921 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 07:58:54.032023 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 07:58:54.041153 27808 raft_consensus.cc:482] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:54.041244 27808 raft_consensus.cc:504] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:54.041513 27808 leader_election.cc:288] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:54.041786 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "057f08ea1ed34b2fb2ca2b56f87b6b3f" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:54.041786 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "057f08ea1ed34b2fb2ca2b56f87b6b3f" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:54.041905 25133 raft_consensus.cc:2454] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:54.041913 24870 raft_consensus.cc:2454] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:54.042152 25199 leader_election.cc:302] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:54.042316 27808 raft_consensus.cc:2781] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:54.042363 27808 raft_consensus.cc:482] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:54.042385 27808 raft_consensus.cc:3037] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.043107 27808 raft_consensus.cc:504] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:54.043394 27808 leader_election.cc:288] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:54.043623 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "057f08ea1ed34b2fb2ca2b56f87b6b3f" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:54.043664 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "057f08ea1ed34b2fb2ca2b56f87b6b3f" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:54.043721 25133 raft_consensus.cc:3037] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.043743 24870 raft_consensus.cc:3037] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.044477 24870 raft_consensus.cc:2454] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:54.044484 25133 raft_consensus.cc:2454] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:54.044745 25202 leader_election.cc:302] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:54.044906 27808 raft_consensus.cc:2781] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:54.044977 27808 raft_consensus.cc:686] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:54.045116 27808 consensus_queue.cc:237] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:54.047081 22440 catalog_manager.cc:5484] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:54.060967 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:54.066256 30333 tablet.cc:1722] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:54.066938 30445 tablet.cc:1722] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:54.067252 30334 tablet.cc:1722] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:54.071914 22442 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=baf1c515beac468790924980777b15ad] alter complete (version 3)
W20240502 07:58:54.099746 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_DELETED RPC failed for tablet 41284d865ebb4cd18f2ce08f66343262: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:54.122532 30234 raft_consensus.cc:482] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:54.122632 30234 raft_consensus.cc:504] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:54.122965 30234 leader_election.cc:288] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:54.123234 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4e349c1038774c0c8f8fcebd1dc3f0c6" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:54.123277 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4e349c1038774c0c8f8fcebd1dc3f0c6" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:54.123335 24870 raft_consensus.cc:2454] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:54.123409 25133 raft_consensus.cc:2454] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:54.123574 22912 leader_election.cc:302] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:54.123705 30234 raft_consensus.cc:2781] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:54.123752 30234 raft_consensus.cc:482] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:54.123788 30234 raft_consensus.cc:3037] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.124890 30234 raft_consensus.cc:504] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:54.125182 30234 leader_election.cc:288] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:54.125411 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4e349c1038774c0c8f8fcebd1dc3f0c6" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:54.125456 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4e349c1038774c0c8f8fcebd1dc3f0c6" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:54.125499 24870 raft_consensus.cc:3037] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.125557 25133 raft_consensus.cc:3037] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.126169 24870 raft_consensus.cc:2454] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:54.126251 25133 raft_consensus.cc:2454] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:54.126382 22912 leader_election.cc:302] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:54.126503 30234 raft_consensus.cc:2781] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:54.126579 30234 raft_consensus.cc:686] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:54.126724 30234 consensus_queue.cc:237] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:54.128289 22440 catalog_manager.cc:5484] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:54.174955 22440 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:54.175100 22440 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:54.176978 22440 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet a46d64c31804473593e0c258e9d807cf
I20240502 07:58:54.177430 24850 tablet_service.cc:1514] Processing DeleteTablet for tablet a46d64c31804473593e0c258e9d807cf with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:54 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:54.177443 25115 tablet_service.cc:1514] Processing DeleteTablet for tablet a46d64c31804473593e0c258e9d807cf with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:54 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:54.177524 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet a46d64c31804473593e0c258e9d807cf with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:54 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:54.177758 30530 tablet_replica.cc:330] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:54.177776 30532 tablet_replica.cc:330] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:54.177898 30530 raft_consensus.cc:2229] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:54.177896 30532 raft_consensus.cc:2229] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:54.178033 30532 raft_consensus.cc:2258] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:54.178040 30530 raft_consensus.cc:2258] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:54.178229 30531 tablet_replica.cc:330] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:54.178400 30531 raft_consensus.cc:2229] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:54.178571 30532 ts_tablet_manager.cc:1875] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:54.178594 30531 raft_consensus.cc:2258] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:54.178648 30530 ts_tablet_manager.cc:1875] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:54.179070 30531 ts_tablet_manager.cc:1875] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:54.180603 30532 ts_tablet_manager.cc:1888] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:54.180773 30532 log.cc:1196] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/a46d64c31804473593e0c258e9d807cf
I20240502 07:58:54.180938 30530 ts_tablet_manager.cc:1888] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:54.181033 30530 log.cc:1196] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/a46d64c31804473593e0c258e9d807cf
I20240502 07:58:54.181178 30531 ts_tablet_manager.cc:1888] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:54.181262 30531 log.cc:1196] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/a46d64c31804473593e0c258e9d807cf
I20240502 07:58:54.181289 30532 ts_tablet_manager.cc:1909] T a46d64c31804473593e0c258e9d807cf P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:54.181321 30530 ts_tablet_manager.cc:1909] T a46d64c31804473593e0c258e9d807cf P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:54.181530 30531 ts_tablet_manager.cc:1909] T a46d64c31804473593e0c258e9d807cf P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:54.181960 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet a46d64c31804473593e0c258e9d807cf (table rebalancer_dynamic_table_new_name [id=baf1c515beac468790924980777b15ad]) successfully deleted
I20240502 07:58:54.182190 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet a46d64c31804473593e0c258e9d807cf (table rebalancer_dynamic_table_new_name [id=baf1c515beac468790924980777b15ad]) successfully deleted
I20240502 07:58:54.182269 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet a46d64c31804473593e0c258e9d807cf (table rebalancer_dynamic_table_new_name [id=baf1c515beac468790924980777b15ad]) successfully deleted
I20240502 07:58:54.278424 22440 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:54.282716 25246 tablet_service.cc:1467] Processing CreateTablet for tablet de975f5a8f9f43ad861b158fb793dced (DEFAULT_TABLE table=rebalancer_dynamic_table [id=f8d3bafc733d4b19884a13430853208f]), partition=
I20240502 07:58:54.282874 22957 tablet_service.cc:1467] Processing CreateTablet for tablet de975f5a8f9f43ad861b158fb793dced (DEFAULT_TABLE table=rebalancer_dynamic_table [id=f8d3bafc733d4b19884a13430853208f]), partition=
I20240502 07:58:54.282917 22827 tablet_service.cc:1467] Processing CreateTablet for tablet de975f5a8f9f43ad861b158fb793dced (DEFAULT_TABLE table=rebalancer_dynamic_table [id=f8d3bafc733d4b19884a13430853208f]), partition=
I20240502 07:58:54.282997 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet de975f5a8f9f43ad861b158fb793dced. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:54.283092 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet de975f5a8f9f43ad861b158fb793dced. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:54.283123 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet de975f5a8f9f43ad861b158fb793dced. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:54.285244 30487 tablet_bootstrap.cc:492] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:54.285245 30491 tablet_bootstrap.cc:492] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:54.285697 30537 tablet_bootstrap.cc:492] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:54.286036 30491 tablet_bootstrap.cc:654] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:54.286036 30487 tablet_bootstrap.cc:654] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:54.286481 30537 tablet_bootstrap.cc:654] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:54.287168 30487 tablet_bootstrap.cc:492] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:54.287249 30487 ts_tablet_manager.cc:1367] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:54.287425 30491 tablet_bootstrap.cc:492] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:54.287508 30491 ts_tablet_manager.cc:1367] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:54.287643 30537 tablet_bootstrap.cc:492] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:54.287711 30537 ts_tablet_manager.cc:1367] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:54.287717 30487 raft_consensus.cc:348] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:54.287822 30487 raft_consensus.cc:374] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:54.287849 30487 raft_consensus.cc:729] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:54.287851 30491 raft_consensus.cc:348] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:54.287925 30491 raft_consensus.cc:374] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:54.287951 30491 raft_consensus.cc:729] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:54.287952 30487 consensus_queue.cc:260] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:54.288049 30491 consensus_queue.cc:260] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:54.288185 30487 ts_tablet_manager.cc:1398] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:54.288295 30491 ts_tablet_manager.cc:1398] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:54.288801 30537 raft_consensus.cc:348] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:54.288973 30537 raft_consensus.cc:374] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:54.289049 30537 raft_consensus.cc:729] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:54.289080 27808 raft_consensus.cc:482] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:54.289152 27808 raft_consensus.cc:504] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:54.289162 30537 consensus_queue.cc:260] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:54.289445 27808 leader_election.cc:288] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:54.290134 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de975f5a8f9f43ad861b158fb793dced" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:54.290251 22847 raft_consensus.cc:2454] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:54.290498 25202 leader_election.cc:302] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:54.290908 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de975f5a8f9f43ad861b158fb793dced" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:54.291018 22979 raft_consensus.cc:2454] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:54.291442 27808 raft_consensus.cc:2781] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:54.291497 27808 raft_consensus.cc:482] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:54.291532 27808 raft_consensus.cc:3037] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.292328 27808 raft_consensus.cc:504] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:54.292934 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de975f5a8f9f43ad861b158fb793dced" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:54.292933 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de975f5a8f9f43ad861b158fb793dced" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:54.293006 22847 raft_consensus.cc:3037] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.293015 22979 raft_consensus.cc:3037] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.293557 22847 raft_consensus.cc:2454] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:54.293656 22979 raft_consensus.cc:2454] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:54.293788 27808 leader_election.cc:288] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:54.293823 25202 leader_election.cc:302] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:54.294014 27808 raft_consensus.cc:2781] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:54.294052 30537 ts_tablet_manager.cc:1398] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:58:54.294095 27808 raft_consensus.cc:686] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:54.294216 27808 consensus_queue.cc:237] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:54.296135 22442 catalog_manager.cc:5484] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:54.335484 30006 raft_consensus.cc:482] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:54.335585 30006 raft_consensus.cc:504] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:54.336412 30006 leader_election.cc:288] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:54.336756 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5d77d4781250465a88338bb4c303918c" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:54.336756 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5d77d4781250465a88338bb4c303918c" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:54.336872 22716 raft_consensus.cc:2454] T 5d77d4781250465a88338bb4c303918c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:54.336872 25266 raft_consensus.cc:2454] T 5d77d4781250465a88338bb4c303918c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:54.337110 22521 leader_election.cc:302] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7eced220cf1642f3873fc385b6191e69, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:54.337262 30006 raft_consensus.cc:2781] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:54.337327 30006 raft_consensus.cc:482] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:54.337363 30006 raft_consensus.cc:3037] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.338258 30006 raft_consensus.cc:504] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:54.338508 30006 leader_election.cc:288] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:54.338753 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5d77d4781250465a88338bb4c303918c" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:54.338841 22716 raft_consensus.cc:3037] T 5d77d4781250465a88338bb4c303918c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.338829 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5d77d4781250465a88338bb4c303918c" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:54.338937 25266 raft_consensus.cc:3037] T 5d77d4781250465a88338bb4c303918c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.339620 22716 raft_consensus.cc:2454] T 5d77d4781250465a88338bb4c303918c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:54.339641 25266 raft_consensus.cc:2454] T 5d77d4781250465a88338bb4c303918c P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:54.339869 22519 leader_election.cc:302] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:54.340013 30006 raft_consensus.cc:2781] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:54.340101 30006 raft_consensus.cc:686] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:54.340229 30006 consensus_queue.cc:237] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:54.341912 22442 catalog_manager.cc:5484] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:54.400161 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:54.403399 22847 raft_consensus.cc:1261] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:54.403733 30573 consensus_queue.cc:1035] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:54.404018 22979 raft_consensus.cc:1261] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:54.404371 30573 consensus_queue.cc:1035] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:54.406394 22442 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet de975f5a8f9f43ad861b158fb793dced (table rebalancer_dynamic_table [id=f8d3bafc733d4b19884a13430853208f]). Expected version 1 got 0
I20240502 07:58:54.406451 30445 tablet.cc:1722] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:54.406728 30445 tablet.cc:1620] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:58:54.406894 30578 tablet.cc:1722] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:54.407166 30578 tablet.cc:1620] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:54.407286 22436 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet de975f5a8f9f43ad861b158fb793dced (table rebalancer_dynamic_table [id=f8d3bafc733d4b19884a13430853208f]). Expected version 1 got 0
I20240502 07:58:54.408483 22442 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet de975f5a8f9f43ad861b158fb793dced (table rebalancer_dynamic_table [id=f8d3bafc733d4b19884a13430853208f]). Expected version 1 got 0
I20240502 07:58:54.409917 30579 tablet.cc:1722] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:54.410935 30579 tablet.cc:1620] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:58:54.413007 22442 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet de975f5a8f9f43ad861b158fb793dced (table rebalancer_dynamic_table [id=f8d3bafc733d4b19884a13430853208f]). Expected version 1 got 0
I20240502 07:58:54.417166 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=f8d3bafc733d4b19884a13430853208f] alter complete (version 1)
W20240502 07:58:54.417863 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:54.449152 27808 consensus_queue.cc:1035] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:54.452901 30573 consensus_queue.cc:1035] T 057f08ea1ed34b2fb2ca2b56f87b6b3f P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:54.521941 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:54.522377 30234 consensus_queue.cc:1035] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:54.526095 30010 consensus_queue.cc:1035] T 4e349c1038774c0c8f8fcebd1dc3f0c6 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:54.527429 30578 tablet.cc:1722] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:54.527709 30578 tablet.cc:1620] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:54.528367 30445 tablet.cc:1722] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:54.528637 30445 tablet.cc:1620] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:58:54.532999 30579 tablet.cc:1722] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:54.533268 30579 tablet.cc:1620] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:58:54.533272 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=f8d3bafc733d4b19884a13430853208f] alter complete (version 2)
W20240502 07:58:54.556186 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:54.635906 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:54.640895 30578 tablet.cc:1722] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:54.641381 30579 tablet.cc:1722] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:54.641789 30445 tablet.cc:1722] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:54.646359 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=f8d3bafc733d4b19884a13430853208f] alter complete (version 3)
W20240502 07:58:54.647308 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 07:58:54.748914 22442 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:54.749037 22442 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:54.750702 22442 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet de975f5a8f9f43ad861b158fb793dced
I20240502 07:58:54.751266 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet de975f5a8f9f43ad861b158fb793dced with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:54 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:54.751272 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet de975f5a8f9f43ad861b158fb793dced with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:54 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:54.751559 30617 tablet_replica.cc:330] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:54.751593 30618 tablet_replica.cc:330] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:54.751576 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet de975f5a8f9f43ad861b158fb793dced with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:54 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:54.751693 30617 raft_consensus.cc:2229] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:54.751722 30618 raft_consensus.cc:2229] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:54.751828 30619 tablet_replica.cc:330] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:54.751842 30617 raft_consensus.cc:2258] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:54.751901 30618 raft_consensus.cc:2258] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:54.752032 30619 raft_consensus.cc:2229] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:54.752238 30619 raft_consensus.cc:2258] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:54.752521 30617 ts_tablet_manager.cc:1875] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:54.752508 30618 ts_tablet_manager.cc:1875] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:54.752818 30619 ts_tablet_manager.cc:1875] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:54.754688 30617 ts_tablet_manager.cc:1888] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:54.754788 30617 log.cc:1196] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/de975f5a8f9f43ad861b158fb793dced
I20240502 07:58:54.754827 30618 ts_tablet_manager.cc:1888] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:54.754840 30619 ts_tablet_manager.cc:1888] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:54.754915 30619 log.cc:1196] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/de975f5a8f9f43ad861b158fb793dced
I20240502 07:58:54.754943 30618 log.cc:1196] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/de975f5a8f9f43ad861b158fb793dced
I20240502 07:58:54.755151 30617 ts_tablet_manager.cc:1909] T de975f5a8f9f43ad861b158fb793dced P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:54.755259 30619 ts_tablet_manager.cc:1909] T de975f5a8f9f43ad861b158fb793dced P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:58:54.755322 30618 ts_tablet_manager.cc:1909] T de975f5a8f9f43ad861b158fb793dced P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:54.755759 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet de975f5a8f9f43ad861b158fb793dced (table rebalancer_dynamic_table_new_name [id=f8d3bafc733d4b19884a13430853208f]) successfully deleted
I20240502 07:58:54.755846 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet de975f5a8f9f43ad861b158fb793dced (table rebalancer_dynamic_table_new_name [id=f8d3bafc733d4b19884a13430853208f]) successfully deleted
I20240502 07:58:54.755856 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet de975f5a8f9f43ad861b158fb793dced (table rebalancer_dynamic_table_new_name [id=f8d3bafc733d4b19884a13430853208f]) successfully deleted
W20240502 07:58:54.814045 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:54.820276 30021 consensus_queue.cc:1035] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:54.824391 30006 consensus_queue.cc:1035] T 5d77d4781250465a88338bb4c303918c P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:58:54.841323 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 07:58:54.843665 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 07:58:54.847564 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:54.852119 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:54.856863 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 1c67e1b1469b4e219d8159782da74bff (DEFAULT_TABLE table=rebalancer_dynamic_table [id=c1b32be6be784892bb5cc058f3c30b25]), partition=
I20240502 07:58:54.856648 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 1c67e1b1469b4e219d8159782da74bff (DEFAULT_TABLE table=rebalancer_dynamic_table [id=c1b32be6be784892bb5cc058f3c30b25]), partition=
I20240502 07:58:54.857121 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c67e1b1469b4e219d8159782da74bff. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:54.857270 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c67e1b1469b4e219d8159782da74bff. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:54.859220 30654 tablet_bootstrap.cc:492] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:54.859237 30655 tablet_bootstrap.cc:492] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:54.859519 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 1c67e1b1469b4e219d8159782da74bff (DEFAULT_TABLE table=rebalancer_dynamic_table [id=c1b32be6be784892bb5cc058f3c30b25]), partition=
I20240502 07:58:54.859723 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c67e1b1469b4e219d8159782da74bff. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:54.859949 30655 tablet_bootstrap.cc:654] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:54.860069 30654 tablet_bootstrap.cc:654] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:54.861001 30654 tablet_bootstrap.cc:492] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:54.861078 30654 ts_tablet_manager.cc:1367] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:54.861490 30654 raft_consensus.cc:348] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:54.861631 30654 raft_consensus.cc:374] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:54.861696 30654 raft_consensus.cc:729] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:54.861761 30658 tablet_bootstrap.cc:492] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:54.861843 30654 consensus_queue.cc:260] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:54.862668 30655 tablet_bootstrap.cc:492] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:54.862586 30658 tablet_bootstrap.cc:654] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:54.862704 30654 ts_tablet_manager.cc:1398] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:54.862784 30655 ts_tablet_manager.cc:1367] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:54.863252 30655 raft_consensus.cc:348] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:54.863345 30655 raft_consensus.cc:374] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:54.863377 30655 raft_consensus.cc:729] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:54.863518 30655 consensus_queue.cc:260] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:54.863797 30655 ts_tablet_manager.cc:1398] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:54.863855 30658 tablet_bootstrap.cc:492] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:54.863986 30658 ts_tablet_manager.cc:1367] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:54.864552 30658 raft_consensus.cc:348] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:54.864667 30658 raft_consensus.cc:374] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:54.864728 30658 raft_consensus.cc:729] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:54.864856 30658 consensus_queue.cc:260] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:54.865301 30658 ts_tablet_manager.cc:1398] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
W20240502 07:58:54.878438 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:54.885171 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_32"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:54.890074 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 5eec173c461a461fa3fa93a014b3ea8d (DEFAULT_TABLE table=rebalancer_extra_table_32 [id=b445b771f9d44c9787bc4a4f78ed6ea5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:54.890327 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5eec173c461a461fa3fa93a014b3ea8d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:54.890427 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 5eec173c461a461fa3fa93a014b3ea8d (DEFAULT_TABLE table=rebalancer_extra_table_32 [id=b445b771f9d44c9787bc4a4f78ed6ea5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:54.890435 25113 tablet_service.cc:1467] Processing CreateTablet for tablet b3f910a4b7304bf39c7136c259d8059e (DEFAULT_TABLE table=rebalancer_extra_table_32 [id=b445b771f9d44c9787bc4a4f78ed6ea5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:54.890632 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5eec173c461a461fa3fa93a014b3ea8d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:54.890736 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b3f910a4b7304bf39c7136c259d8059e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:54.892822 22695 tablet_service.cc:1467] Processing CreateTablet for tablet b3f910a4b7304bf39c7136c259d8059e (DEFAULT_TABLE table=rebalancer_extra_table_32 [id=b445b771f9d44c9787bc4a4f78ed6ea5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:54.892925 30660 tablet_bootstrap.cc:492] T 5eec173c461a461fa3fa93a014b3ea8d P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:54.892917 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 5eec173c461a461fa3fa93a014b3ea8d (DEFAULT_TABLE table=rebalancer_extra_table_32 [id=b445b771f9d44c9787bc4a4f78ed6ea5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:54.893018 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b3f910a4b7304bf39c7136c259d8059e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:54.893157 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5eec173c461a461fa3fa93a014b3ea8d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:54.893780 30660 tablet_bootstrap.cc:654] T 5eec173c461a461fa3fa93a014b3ea8d P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:54.894837 22827 tablet_service.cc:1467] Processing CreateTablet for tablet b3f910a4b7304bf39c7136c259d8059e (DEFAULT_TABLE table=rebalancer_extra_table_32 [id=b445b771f9d44c9787bc4a4f78ed6ea5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:54.895095 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b3f910a4b7304bf39c7136c259d8059e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:54.896119 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 82829aa237874f7592f337f77fbd58ab (DEFAULT_TABLE table=rebalancer_extra_table_32 [id=b445b771f9d44c9787bc4a4f78ed6ea5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:54.896342 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 82829aa237874f7592f337f77fbd58ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:54.896433 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 82829aa237874f7592f337f77fbd58ab (DEFAULT_TABLE table=rebalancer_extra_table_32 [id=b445b771f9d44c9787bc4a4f78ed6ea5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:54.896612 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 82829aa237874f7592f337f77fbd58ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:54.898181 30006 raft_consensus.cc:482] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:54.898259 30006 raft_consensus.cc:504] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:54.898542 30006 leader_election.cc:288] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:54.899173 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c67e1b1469b4e219d8159782da74bff" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:54.899266 25133 raft_consensus.cc:2454] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:54.899364 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c67e1b1469b4e219d8159782da74bff" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:54.899477 25265 raft_consensus.cc:2454] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:54.899571 22522 leader_election.cc:302] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:54.899757 30006 raft_consensus.cc:2781] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:54.899799 30006 raft_consensus.cc:482] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:54.899827 30006 raft_consensus.cc:3037] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.900091 30655 tablet_bootstrap.cc:492] T b3f910a4b7304bf39c7136c259d8059e P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:54.900866 30655 tablet_bootstrap.cc:654] T b3f910a4b7304bf39c7136c259d8059e P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:54.900650 30006 raft_consensus.cc:504] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:54.901180 30006 leader_election.cc:288] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:54.901474 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c67e1b1469b4e219d8159782da74bff" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:54.901552 25265 raft_consensus.cc:3037] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.901779 30655 tablet_bootstrap.cc:492] T b3f910a4b7304bf39c7136c259d8059e P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:54.901836 30655 ts_tablet_manager.cc:1367] T b3f910a4b7304bf39c7136c259d8059e P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:54.902308 25265 raft_consensus.cc:2454] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:54.902235 30655 raft_consensus.cc:348] T b3f910a4b7304bf39c7136c259d8059e P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:54.902544 22521 leader_election.cc:302] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7eced220cf1642f3873fc385b6191e69, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
W20240502 07:58:54.902590 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:54.902674 30655 raft_consensus.cc:374] T b3f910a4b7304bf39c7136c259d8059e P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:54.902714 30655 raft_consensus.cc:729] T b3f910a4b7304bf39c7136c259d8059e P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:54.902825 30655 consensus_queue.cc:260] T b3f910a4b7304bf39c7136c259d8059e P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:54.903038 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c67e1b1469b4e219d8159782da74bff" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:54.903126 25133 raft_consensus.cc:3037] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.903697 30660 tablet_bootstrap.cc:492] T 5eec173c461a461fa3fa93a014b3ea8d P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:54.903766 30660 ts_tablet_manager.cc:1367] T 5eec173c461a461fa3fa93a014b3ea8d P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.000s
I20240502 07:58:54.903859 25133 raft_consensus.cc:2454] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:54.904152 30660 raft_consensus.cc:348] T 5eec173c461a461fa3fa93a014b3ea8d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:54.904289 30006 raft_consensus.cc:2781] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:54.904300 30660 raft_consensus.cc:374] T 5eec173c461a461fa3fa93a014b3ea8d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:54.904379 30006 raft_consensus.cc:686] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:54.904479 30660 raft_consensus.cc:729] T 5eec173c461a461fa3fa93a014b3ea8d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:54.904516 30006 consensus_queue.cc:237] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:54.904729 30660 consensus_queue.cc:260] T 5eec173c461a461fa3fa93a014b3ea8d P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:54.905107 30660 ts_tablet_manager.cc:1398] T 5eec173c461a461fa3fa93a014b3ea8d P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:54.905419 30660 tablet_bootstrap.cc:492] T b3f910a4b7304bf39c7136c259d8059e P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:54.906111 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 82829aa237874f7592f337f77fbd58ab (DEFAULT_TABLE table=rebalancer_extra_table_32 [id=b445b771f9d44c9787bc4a4f78ed6ea5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:54.906320 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 82829aa237874f7592f337f77fbd58ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:54.906459 30660 tablet_bootstrap.cc:654] T b3f910a4b7304bf39c7136c259d8059e P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:54.907081 30662 tablet_bootstrap.cc:492] T 5eec173c461a461fa3fa93a014b3ea8d P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:54.907907 30662 tablet_bootstrap.cc:654] T 5eec173c461a461fa3fa93a014b3ea8d P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:54.903065 30655 ts_tablet_manager.cc:1398] T b3f910a4b7304bf39c7136c259d8059e P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:54.909025 30655 tablet_bootstrap.cc:492] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:54.909915 30655 tablet_bootstrap.cc:654] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:54.911119 30655 tablet_bootstrap.cc:492] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:54.911193 30655 ts_tablet_manager.cc:1367] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:54.911578 30655 raft_consensus.cc:348] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:54.911665 30655 raft_consensus.cc:374] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:54.911733 30655 raft_consensus.cc:729] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:54.911657 22442 catalog_manager.cc:5484] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:54.911876 30662 tablet_bootstrap.cc:492] T 5eec173c461a461fa3fa93a014b3ea8d P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:54.911844 30655 consensus_queue.cc:260] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:54.911952 30662 ts_tablet_manager.cc:1367] T 5eec173c461a461fa3fa93a014b3ea8d P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
I20240502 07:58:54.912097 30655 ts_tablet_manager.cc:1398] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:54.912420 30662 raft_consensus.cc:348] T 5eec173c461a461fa3fa93a014b3ea8d P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:54.912513 30662 raft_consensus.cc:374] T 5eec173c461a461fa3fa93a014b3ea8d P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:54.912546 30662 raft_consensus.cc:729] T 5eec173c461a461fa3fa93a014b3ea8d P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:54.912811 30658 tablet_bootstrap.cc:492] T 82829aa237874f7592f337f77fbd58ab P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:54.913157 30662 consensus_queue.cc:260] T 5eec173c461a461fa3fa93a014b3ea8d P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:54.913542 30662 ts_tablet_manager.cc:1398] T 5eec173c461a461fa3fa93a014b3ea8d P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:54.913856 30658 tablet_bootstrap.cc:654] T 82829aa237874f7592f337f77fbd58ab P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:54.919201 30658 tablet_bootstrap.cc:492] T 82829aa237874f7592f337f77fbd58ab P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:54.919301 30658 ts_tablet_manager.cc:1367] T 82829aa237874f7592f337f77fbd58ab P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
W20240502 07:58:54.919381 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:54.919644 30658 raft_consensus.cc:348] T 82829aa237874f7592f337f77fbd58ab P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:54.919734 30658 raft_consensus.cc:374] T 82829aa237874f7592f337f77fbd58ab P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:54.919764 30658 raft_consensus.cc:729] T 82829aa237874f7592f337f77fbd58ab P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:54.919847 30658 consensus_queue.cc:260] T 82829aa237874f7592f337f77fbd58ab P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:54.920066 30658 ts_tablet_manager.cc:1398] T 82829aa237874f7592f337f77fbd58ab P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:54.925420 30660 tablet_bootstrap.cc:492] T b3f910a4b7304bf39c7136c259d8059e P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:54.925493 30660 ts_tablet_manager.cc:1367] T b3f910a4b7304bf39c7136c259d8059e P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.020s user 0.002s sys 0.000s
I20240502 07:58:54.925817 30660 raft_consensus.cc:348] T b3f910a4b7304bf39c7136c259d8059e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:54.925901 30660 raft_consensus.cc:374] T b3f910a4b7304bf39c7136c259d8059e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:54.925922 30660 raft_consensus.cc:729] T b3f910a4b7304bf39c7136c259d8059e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:54.926005 30660 consensus_queue.cc:260] T b3f910a4b7304bf39c7136c259d8059e P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:54.926249 30660 ts_tablet_manager.cc:1398] T b3f910a4b7304bf39c7136c259d8059e P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:54.926367 30660 tablet_bootstrap.cc:492] T 82829aa237874f7592f337f77fbd58ab P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:54.926992 30660 tablet_bootstrap.cc:654] T 82829aa237874f7592f337f77fbd58ab P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:54.927861 30660 tablet_bootstrap.cc:492] T 82829aa237874f7592f337f77fbd58ab P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:54.927930 30660 ts_tablet_manager.cc:1367] T 82829aa237874f7592f337f77fbd58ab P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:54.928269 30660 raft_consensus.cc:348] T 82829aa237874f7592f337f77fbd58ab P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:54.928345 30660 raft_consensus.cc:374] T 82829aa237874f7592f337f77fbd58ab P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:54.928370 30660 raft_consensus.cc:729] T 82829aa237874f7592f337f77fbd58ab P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:54.928464 30660 consensus_queue.cc:260] T 82829aa237874f7592f337f77fbd58ab P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:54.928822 30660 ts_tablet_manager.cc:1398] T 82829aa237874f7592f337f77fbd58ab P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:54.928838 30663 tablet_bootstrap.cc:492] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:54.929544 30663 tablet_bootstrap.cc:654] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:54.932633 30663 tablet_bootstrap.cc:492] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:54.932760 30663 ts_tablet_manager.cc:1367] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:54.933218 30663 raft_consensus.cc:348] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:54.933320 30663 raft_consensus.cc:374] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:54.933349 30663 raft_consensus.cc:729] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:54.933471 30663 consensus_queue.cc:260] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:54.933732 30663 ts_tablet_manager.cc:1398] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:54.933869 30663 tablet_bootstrap.cc:492] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:54.934760 30663 tablet_bootstrap.cc:654] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:54.936256 30663 tablet_bootstrap.cc:492] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:54.936362 30663 ts_tablet_manager.cc:1367] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:54.936769 30663 raft_consensus.cc:348] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:54.936892 30663 raft_consensus.cc:374] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:54.936949 30663 raft_consensus.cc:729] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:54.937079 30663 consensus_queue.cc:260] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:54.937361 30663 ts_tablet_manager.cc:1398] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:54.952265 30455 raft_consensus.cc:482] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:54.952332 30455 raft_consensus.cc:504] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:54.952700 30455 leader_election.cc:288] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:54.952929 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "82829aa237874f7592f337f77fbd58ab" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:54.952942 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "82829aa237874f7592f337f77fbd58ab" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:54.953019 22716 raft_consensus.cc:2454] T 82829aa237874f7592f337f77fbd58ab P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:54.953028 25265 raft_consensus.cc:2454] T 82829aa237874f7592f337f77fbd58ab P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:54.953220 22783 leader_election.cc:302] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:54.953341 30455 raft_consensus.cc:2781] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:54.953385 30455 raft_consensus.cc:482] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:54.953408 30455 raft_consensus.cc:3037] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.954126 30455 raft_consensus.cc:504] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:54.954375 30455 leader_election.cc:288] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:54.954632 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "82829aa237874f7592f337f77fbd58ab" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:54.954648 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "82829aa237874f7592f337f77fbd58ab" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:54.954738 25265 raft_consensus.cc:3037] T 82829aa237874f7592f337f77fbd58ab P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.954738 22716 raft_consensus.cc:3037] T 82829aa237874f7592f337f77fbd58ab P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:54.955574 25265 raft_consensus.cc:2454] T 82829aa237874f7592f337f77fbd58ab P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:54.955574 22716 raft_consensus.cc:2454] T 82829aa237874f7592f337f77fbd58ab P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:54.955804 22781 leader_election.cc:302] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:54.955943 30455 raft_consensus.cc:2781] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:54.956017 30455 raft_consensus.cc:686] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:54.956125 30455 consensus_queue.cc:237] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:54.957664 22436 catalog_manager.cc:5484] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:54.985906 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 82 similar messages]
I20240502 07:58:55.019716 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:55.023313 25265 raft_consensus.cc:1261] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:55.023597 25135 raft_consensus.cc:1261] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:55.023988 30006 consensus_queue.cc:1035] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:55.024600 30006 consensus_queue.cc:1035] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:55.027125 22436 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 1c67e1b1469b4e219d8159782da74bff (table rebalancer_dynamic_table [id=c1b32be6be784892bb5cc058f3c30b25]). Expected version 1 got 0
I20240502 07:58:55.027637 30578 tablet.cc:1722] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:55.027915 30578 tablet.cc:1620] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:55.029284 30709 tablet.cc:1722] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:55.029765 30709 tablet.cc:1620] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:58:55.028623 22442 catalog_manager.cc:5576] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451) does not have the latest schema for tablet 1c67e1b1469b4e219d8159782da74bff (table rebalancer_dynamic_table [id=c1b32be6be784892bb5cc058f3c30b25]). Expected version 1 got 0
I20240502 07:58:55.030472 22436 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet 1c67e1b1469b4e219d8159782da74bff (table rebalancer_dynamic_table [id=c1b32be6be784892bb5cc058f3c30b25]). Expected version 1 got 0
I20240502 07:58:55.030830 22440 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 1c67e1b1469b4e219d8159782da74bff (table rebalancer_dynamic_table [id=c1b32be6be784892bb5cc058f3c30b25]). Expected version 1 got 0
I20240502 07:58:55.032171 30710 tablet.cc:1722] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:55.032466 30710 tablet.cc:1620] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:55.035095 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=c1b32be6be784892bb5cc058f3c30b25] alter complete (version 1)
W20240502 07:58:55.052971 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 07:58:55.053503 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:55.061390 30455 raft_consensus.cc:482] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:55.061472 30455 raft_consensus.cc:504] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:55.061765 30455 leader_election.cc:288] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:55.062021 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3f910a4b7304bf39c7136c259d8059e" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:55.062047 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3f910a4b7304bf39c7136c259d8059e" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:55.062160 25135 raft_consensus.cc:2454] T b3f910a4b7304bf39c7136c259d8059e P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:55.062165 22716 raft_consensus.cc:2454] T b3f910a4b7304bf39c7136c259d8059e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:55.062387 22784 leader_election.cc:302] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:55.062565 30107 raft_consensus.cc:2781] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:55.062628 30107 raft_consensus.cc:482] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:55.062662 30107 raft_consensus.cc:3037] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.063515 30107 raft_consensus.cc:504] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:55.063812 30107 leader_election.cc:288] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:55.064148 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3f910a4b7304bf39c7136c259d8059e" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:55.064256 22716 raft_consensus.cc:3037] T b3f910a4b7304bf39c7136c259d8059e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.064241 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3f910a4b7304bf39c7136c259d8059e" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:55.064363 25135 raft_consensus.cc:3037] T b3f910a4b7304bf39c7136c259d8059e P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.065145 22716 raft_consensus.cc:2454] T b3f910a4b7304bf39c7136c259d8059e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:55.065214 25135 raft_consensus.cc:2454] T b3f910a4b7304bf39c7136c259d8059e P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:55.065387 22781 leader_election.cc:302] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:55.065552 30107 raft_consensus.cc:2781] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:55.065630 30107 raft_consensus.cc:686] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:55.065758 30107 consensus_queue.cc:237] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:55.067348 22436 catalog_manager.cc:5484] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:55.079063 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 07:58:55.086056 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 64 similar messages]
I20240502 07:58:55.113390 30669 raft_consensus.cc:482] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:55.113489 30669 raft_consensus.cc:504] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:55.113884 30669 leader_election.cc:288] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:55.114184 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5eec173c461a461fa3fa93a014b3ea8d" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:55.114249 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5eec173c461a461fa3fa93a014b3ea8d" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:55.114322 22716 raft_consensus.cc:2454] T 5eec173c461a461fa3fa93a014b3ea8d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:55.114372 24870 raft_consensus.cc:2454] T 5eec173c461a461fa3fa93a014b3ea8d P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:55.114576 25068 leader_election.cc:302] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:55.114727 30669 raft_consensus.cc:2781] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:55.114791 30669 raft_consensus.cc:482] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:55.114830 30669 raft_consensus.cc:3037] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.115653 30669 raft_consensus.cc:504] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:55.115883 30669 leader_election.cc:288] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:55.116140 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5eec173c461a461fa3fa93a014b3ea8d" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:55.116182 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5eec173c461a461fa3fa93a014b3ea8d" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:55.116235 22716 raft_consensus.cc:3037] T 5eec173c461a461fa3fa93a014b3ea8d P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.116283 24870 raft_consensus.cc:3037] T 5eec173c461a461fa3fa93a014b3ea8d P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.116999 22716 raft_consensus.cc:2454] T 5eec173c461a461fa3fa93a014b3ea8d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:55.117173 24870 raft_consensus.cc:2454] T 5eec173c461a461fa3fa93a014b3ea8d P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:55.117236 25068 leader_election.cc:302] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:55.117370 30669 raft_consensus.cc:2781] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:55.117446 30669 raft_consensus.cc:686] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:58:55.117561 30669 consensus_queue.cc:237] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:55.119580 22436 catalog_manager.cc:5484] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:55.136204 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
W20240502 07:58:55.139837 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
I20240502 07:58:55.140762 30709 tablet.cc:1722] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:55.141048 30709 tablet.cc:1620] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:58:55.141422 30578 tablet.cc:1722] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:55.141664 30578 tablet.cc:1620] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:55.143057 30710 tablet.cc:1722] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:55.143332 30710 tablet.cc:1620] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:55.146013 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=c1b32be6be784892bb5cc058f3c30b25] alter complete (version 2)
W20240502 07:58:55.160049 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 07:58:55.179437 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 07:58:55.181212 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 35 similar messages]
W20240502 07:58:55.218446 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 07:58:55.233681 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 07:58:55.240996 22430 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 114 similar messages]
W20240502 07:58:55.242486 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): CreateTablet RPC failed for tablet 68cfb914f8c140728ba025b37d7da7f5: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:55.249363 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:55.260154 30709 tablet.cc:1722] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:55.261047 30578 tablet.cc:1722] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:55.263187 30710 tablet.cc:1722] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:55.266206 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=c1b32be6be784892bb5cc058f3c30b25] alter complete (version 3)
W20240502 07:58:55.356108 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 07:58:55.367982 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:55.372474 22444 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:55.372606 22444 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:55.375105 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 1c67e1b1469b4e219d8159782da74bff
I20240502 07:58:55.375605 25115 tablet_service.cc:1514] Processing DeleteTablet for tablet 1c67e1b1469b4e219d8159782da74bff with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:55 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:55.375874 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 1c67e1b1469b4e219d8159782da74bff with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:55 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:55.376091 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet 1c67e1b1469b4e219d8159782da74bff with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:55 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:55.376171 30748 tablet_replica.cc:330] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:55.376298 30748 raft_consensus.cc:2229] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:55.376487 30748 raft_consensus.cc:2258] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:55.376711 30747 tablet_replica.cc:330] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:55.376915 30747 raft_consensus.cc:2229] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:55.377089 30747 raft_consensus.cc:2258] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:55.377674 30748 ts_tablet_manager.cc:1875] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:55.377671 30747 ts_tablet_manager.cc:1875] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:55.379356 30749 tablet_replica.cc:330] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:55.379515 30749 raft_consensus.cc:2229] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:55.379666 30749 raft_consensus.cc:2258] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:55.379776 30748 ts_tablet_manager.cc:1888] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:55.379776 30747 ts_tablet_manager.cc:1888] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:55.379844 30747 log.cc:1196] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/1c67e1b1469b4e219d8159782da74bff
I20240502 07:58:55.379865 30748 log.cc:1196] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/1c67e1b1469b4e219d8159782da74bff
I20240502 07:58:55.380169 30747 ts_tablet_manager.cc:1909] T 1c67e1b1469b4e219d8159782da74bff P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:55.380179 30748 ts_tablet_manager.cc:1909] T 1c67e1b1469b4e219d8159782da74bff P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:55.380275 30749 ts_tablet_manager.cc:1875] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:55.380820 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 1c67e1b1469b4e219d8159782da74bff (table rebalancer_dynamic_table_new_name [id=c1b32be6be784892bb5cc058f3c30b25]) successfully deleted
I20240502 07:58:55.381029 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet 1c67e1b1469b4e219d8159782da74bff (table rebalancer_dynamic_table_new_name [id=c1b32be6be784892bb5cc058f3c30b25]) successfully deleted
I20240502 07:58:55.382150 30749 ts_tablet_manager.cc:1888] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:55.382203 30749 log.cc:1196] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/1c67e1b1469b4e219d8159782da74bff
I20240502 07:58:55.382398 30749 ts_tablet_manager.cc:1909] T 1c67e1b1469b4e219d8159782da74bff P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:58:55.382853 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet 1c67e1b1469b4e219d8159782da74bff (table rebalancer_dynamic_table_new_name [id=c1b32be6be784892bb5cc058f3c30b25]) successfully deleted
W20240502 07:58:55.407950 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:58:55.429644 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20240502 07:58:55.430140 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 07:58:55.430701 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 07:58:55.452925 30107 consensus_queue.cc:1035] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:55.455825 30455 consensus_queue.cc:1035] T b3f910a4b7304bf39c7136c259d8059e P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:55.477312 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:55.482039 22957 tablet_service.cc:1467] Processing CreateTablet for tablet d61d7974ed6144628d1cddb87122abdf (DEFAULT_TABLE table=rebalancer_dynamic_table [id=7cc1034a6792451a96febbce6db220ee]), partition=
I20240502 07:58:55.482291 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d61d7974ed6144628d1cddb87122abdf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:55.482345 25246 tablet_service.cc:1467] Processing CreateTablet for tablet d61d7974ed6144628d1cddb87122abdf (DEFAULT_TABLE table=rebalancer_dynamic_table [id=7cc1034a6792451a96febbce6db220ee]), partition=
I20240502 07:58:55.482528 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d61d7974ed6144628d1cddb87122abdf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:55.484534 22566 tablet_service.cc:1467] Processing CreateTablet for tablet d61d7974ed6144628d1cddb87122abdf (DEFAULT_TABLE table=rebalancer_dynamic_table [id=7cc1034a6792451a96febbce6db220ee]), partition=
I20240502 07:58:55.484699 30455 consensus_queue.cc:1035] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:55.484783 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d61d7974ed6144628d1cddb87122abdf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:55.484777 30769 tablet_bootstrap.cc:492] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:55.485744 30769 tablet_bootstrap.cc:654] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:55.486598 30773 tablet_bootstrap.cc:492] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:55.486852 30769 tablet_bootstrap.cc:492] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:55.486912 30769 ts_tablet_manager.cc:1367] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:55.487215 30769 raft_consensus.cc:348] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:55.487279 30769 raft_consensus.cc:374] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:55.487296 30769 raft_consensus.cc:729] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:55.487323 30773 tablet_bootstrap.cc:654] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:55.487385 30769 consensus_queue.cc:260] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:55.487591 30769 ts_tablet_manager.cc:1398] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:55.489225 30770 tablet_bootstrap.cc:492] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:55.489571 30773 tablet_bootstrap.cc:492] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:55.489684 30773 ts_tablet_manager.cc:1367] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:55.490100 30773 raft_consensus.cc:348] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:55.490253 30773 raft_consensus.cc:374] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:55.490314 30770 tablet_bootstrap.cc:654] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:55.490320 30773 raft_consensus.cc:729] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:55.490443 30773 consensus_queue.cc:260] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:55.491293 30770 tablet_bootstrap.cc:492] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:55.491369 30770 ts_tablet_manager.cc:1367] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:55.491588 30107 consensus_queue.cc:1035] T 82829aa237874f7592f337f77fbd58ab P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:55.491746 30770 raft_consensus.cc:348] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:55.491835 30770 raft_consensus.cc:374] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:55.491869 30770 raft_consensus.cc:729] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:55.491909 30773 ts_tablet_manager.cc:1398] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:55.491978 30770 consensus_queue.cc:260] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:55.492239 30770 ts_tablet_manager.cc:1398] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:55.497124 30669 consensus_queue.cc:1035] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:55.500173 30783 consensus_queue.cc:1035] T 5eec173c461a461fa3fa93a014b3ea8d P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:55.640520 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_34"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:55.646169 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 1c2f5da964fe4ab2bbd71539fdf7a75e (DEFAULT_TABLE table=rebalancer_extra_table_34 [id=1c68ccb9da7e4b2eb1ed75646152391a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:55.646201 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 1c2f5da964fe4ab2bbd71539fdf7a75e (DEFAULT_TABLE table=rebalancer_extra_table_34 [id=1c68ccb9da7e4b2eb1ed75646152391a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:55.646441 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c2f5da964fe4ab2bbd71539fdf7a75e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:55.646442 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c2f5da964fe4ab2bbd71539fdf7a75e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:55.648736 30769 tablet_bootstrap.cc:492] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:55.649437 22828 tablet_service.cc:1467] Processing CreateTablet for tablet aedd9079c95746998d6b50a15fa96a9a (DEFAULT_TABLE table=rebalancer_extra_table_34 [id=1c68ccb9da7e4b2eb1ed75646152391a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:55.649668 30796 tablet_bootstrap.cc:492] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:55.649703 30769 tablet_bootstrap.cc:654] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:55.649739 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aedd9079c95746998d6b50a15fa96a9a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:55.650666 30796 tablet_bootstrap.cc:654] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:55.650661 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 1c2f5da964fe4ab2bbd71539fdf7a75e (DEFAULT_TABLE table=rebalancer_extra_table_34 [id=1c68ccb9da7e4b2eb1ed75646152391a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:55.650944 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c2f5da964fe4ab2bbd71539fdf7a75e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:55.651758 30796 tablet_bootstrap.cc:492] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:55.651880 30796 ts_tablet_manager.cc:1367] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:55.652345 30796 raft_consensus.cc:348] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:55.652493 30796 raft_consensus.cc:374] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:55.652556 30796 raft_consensus.cc:729] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:55.652740 30796 consensus_queue.cc:260] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:55.652922 25115 tablet_service.cc:1467] Processing CreateTablet for tablet aedd9079c95746998d6b50a15fa96a9a (DEFAULT_TABLE table=rebalancer_extra_table_34 [id=1c68ccb9da7e4b2eb1ed75646152391a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:55.653172 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aedd9079c95746998d6b50a15fa96a9a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:55.653278 30796 ts_tablet_manager.cc:1398] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:55.655146 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 100d4b30eb344356943eb8cbe2c28bf2 (DEFAULT_TABLE table=rebalancer_extra_table_34 [id=1c68ccb9da7e4b2eb1ed75646152391a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:55.655356 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 100d4b30eb344356943eb8cbe2c28bf2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:55.655655 30800 tablet_bootstrap.cc:492] T aedd9079c95746998d6b50a15fa96a9a P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:55.656502 30800 tablet_bootstrap.cc:654] T aedd9079c95746998d6b50a15fa96a9a P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:55.657632 30800 tablet_bootstrap.cc:492] T aedd9079c95746998d6b50a15fa96a9a P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:55.657716 30800 ts_tablet_manager.cc:1367] T aedd9079c95746998d6b50a15fa96a9a P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:58:55.658214 30800 raft_consensus.cc:348] T aedd9079c95746998d6b50a15fa96a9a P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:55.658335 30800 raft_consensus.cc:374] T aedd9079c95746998d6b50a15fa96a9a P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:55.658391 30800 raft_consensus.cc:729] T aedd9079c95746998d6b50a15fa96a9a P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:55.658455 24850 tablet_service.cc:1467] Processing CreateTablet for tablet aedd9079c95746998d6b50a15fa96a9a (DEFAULT_TABLE table=rebalancer_extra_table_34 [id=1c68ccb9da7e4b2eb1ed75646152391a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:55.658535 30800 consensus_queue.cc:260] T aedd9079c95746998d6b50a15fa96a9a P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:55.658720 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aedd9079c95746998d6b50a15fa96a9a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:55.658905 30800 ts_tablet_manager.cc:1398] T aedd9079c95746998d6b50a15fa96a9a P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:55.659039 30800 tablet_bootstrap.cc:492] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:55.659083 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 100d4b30eb344356943eb8cbe2c28bf2 (DEFAULT_TABLE table=rebalancer_extra_table_34 [id=1c68ccb9da7e4b2eb1ed75646152391a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:55.659260 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 100d4b30eb344356943eb8cbe2c28bf2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:55.659888 30800 tablet_bootstrap.cc:654] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:55.660704 30769 tablet_bootstrap.cc:492] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:55.660792 30769 ts_tablet_manager.cc:1367] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.012s user 0.002s sys 0.000s
I20240502 07:58:55.661134 30803 tablet_bootstrap.cc:492] T 100d4b30eb344356943eb8cbe2c28bf2 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:55.661216 30769 raft_consensus.cc:348] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:55.661300 30769 raft_consensus.cc:374] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:55.661330 30769 raft_consensus.cc:729] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:55.661434 30769 consensus_queue.cc:260] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:55.661911 30803 tablet_bootstrap.cc:654] T 100d4b30eb344356943eb8cbe2c28bf2 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:55.662118 30769 ts_tablet_manager.cc:1398] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:55.663007 30773 tablet_bootstrap.cc:492] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:55.663264 30803 tablet_bootstrap.cc:492] T 100d4b30eb344356943eb8cbe2c28bf2 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:55.663364 30803 ts_tablet_manager.cc:1367] T 100d4b30eb344356943eb8cbe2c28bf2 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:55.663357 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 100d4b30eb344356943eb8cbe2c28bf2 (DEFAULT_TABLE table=rebalancer_extra_table_34 [id=1c68ccb9da7e4b2eb1ed75646152391a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:55.663625 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 100d4b30eb344356943eb8cbe2c28bf2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:55.663779 30803 raft_consensus.cc:348] T 100d4b30eb344356943eb8cbe2c28bf2 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:55.663878 30803 raft_consensus.cc:374] T 100d4b30eb344356943eb8cbe2c28bf2 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:55.663909 30803 raft_consensus.cc:729] T 100d4b30eb344356943eb8cbe2c28bf2 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:55.664063 30773 tablet_bootstrap.cc:654] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:55.664013 30803 consensus_queue.cc:260] T 100d4b30eb344356943eb8cbe2c28bf2 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:55.664271 30803 ts_tablet_manager.cc:1398] T 100d4b30eb344356943eb8cbe2c28bf2 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:55.664422 30803 tablet_bootstrap.cc:492] T aedd9079c95746998d6b50a15fa96a9a P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:55.665324 30803 tablet_bootstrap.cc:654] T aedd9079c95746998d6b50a15fa96a9a P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:55.665474 30685 raft_consensus.cc:482] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:55.665535 30685 raft_consensus.cc:504] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:55.666635 30769 tablet_bootstrap.cc:492] T 100d4b30eb344356943eb8cbe2c28bf2 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:55.666832 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c2f5da964fe4ab2bbd71539fdf7a75e" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:55.667006 22716 raft_consensus.cc:2454] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:55.667397 30769 tablet_bootstrap.cc:654] T 100d4b30eb344356943eb8cbe2c28bf2 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:55.667268 22912 leader_election.cc:302] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:55.667671 30685 leader_election.cc:288] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:55.667860 30685 raft_consensus.cc:2781] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:55.668011 30685 raft_consensus.cc:482] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:55.668171 30685 raft_consensus.cc:3037] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.668305 30799 tablet_bootstrap.cc:492] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:55.669596 30799 tablet_bootstrap.cc:654] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:55.670046 30800 tablet_bootstrap.cc:492] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:55.670125 30800 ts_tablet_manager.cc:1367] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.011s user 0.000s sys 0.002s
I20240502 07:58:55.670552 30803 tablet_bootstrap.cc:492] T aedd9079c95746998d6b50a15fa96a9a P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:55.670611 30803 ts_tablet_manager.cc:1367] T aedd9079c95746998d6b50a15fa96a9a P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:58:55.670634 30800 raft_consensus.cc:348] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:55.670722 30800 raft_consensus.cc:374] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:55.670748 30800 raft_consensus.cc:729] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:55.671067 30803 raft_consensus.cc:348] T aedd9079c95746998d6b50a15fa96a9a P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:55.671202 30803 raft_consensus.cc:374] T aedd9079c95746998d6b50a15fa96a9a P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:55.671269 30803 raft_consensus.cc:729] T aedd9079c95746998d6b50a15fa96a9a P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:55.671411 30803 consensus_queue.cc:260] T aedd9079c95746998d6b50a15fa96a9a P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:55.671773 30803 ts_tablet_manager.cc:1398] T aedd9079c95746998d6b50a15fa96a9a P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:55.671875 30685 raft_consensus.cc:504] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:55.672227 30685 leader_election.cc:288] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:55.672451 30800 consensus_queue.cc:260] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:55.672557 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c2f5da964fe4ab2bbd71539fdf7a75e" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:55.672688 22716 raft_consensus.cc:3037] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.672924 30800 ts_tablet_manager.cc:1398] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.003s user 0.000s sys 0.001s
I20240502 07:58:55.673508 22716 raft_consensus.cc:2454] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:55.673791 22912 leader_election.cc:302] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:55.674571 30773 tablet_bootstrap.cc:492] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:55.674665 30773 ts_tablet_manager.cc:1367] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.012s user 0.002s sys 0.000s
I20240502 07:58:55.674886 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c2f5da964fe4ab2bbd71539fdf7a75e" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:55.675109 30773 raft_consensus.cc:348] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:55.675211 30773 raft_consensus.cc:374] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:55.675261 30773 raft_consensus.cc:729] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:55.675369 30773 consensus_queue.cc:260] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:55.675601 30773 ts_tablet_manager.cc:1398] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:55.676118 30807 raft_consensus.cc:2781] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:55.676206 30807 raft_consensus.cc:686] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:55.675207 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c2f5da964fe4ab2bbd71539fdf7a75e" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:55.676337 30807 consensus_queue.cc:237] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:55.676414 22584 raft_consensus.cc:3037] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.676215 30769 tablet_bootstrap.cc:492] T 100d4b30eb344356943eb8cbe2c28bf2 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:55.676544 30769 ts_tablet_manager.cc:1367] T 100d4b30eb344356943eb8cbe2c28bf2 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.000s
I20240502 07:58:55.676968 30769 raft_consensus.cc:348] T 100d4b30eb344356943eb8cbe2c28bf2 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:55.677067 30769 raft_consensus.cc:374] T 100d4b30eb344356943eb8cbe2c28bf2 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:55.677103 30769 raft_consensus.cc:729] T 100d4b30eb344356943eb8cbe2c28bf2 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:55.677230 30769 consensus_queue.cc:260] T 100d4b30eb344356943eb8cbe2c28bf2 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:55.677327 22584 raft_consensus.cc:2454] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:55.677443 30769 ts_tablet_manager.cc:1398] T 100d4b30eb344356943eb8cbe2c28bf2 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
W20240502 07:58:55.678095 22915 leader_election.cc:341] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): Illegal state: must be running to vote when last-logged opid is not known
I20240502 07:58:55.679195 22442 catalog_manager.cc:5484] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:55.679579 30799 tablet_bootstrap.cc:492] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:55.679710 30799 ts_tablet_manager.cc:1367] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.000s
I20240502 07:58:55.680145 30799 raft_consensus.cc:348] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:55.680289 30799 raft_consensus.cc:374] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:55.680361 30799 raft_consensus.cc:729] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:55.680568 30799 consensus_queue.cc:260] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:55.683501 30799 ts_tablet_manager.cc:1398] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:55.685581 30006 raft_consensus.cc:482] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:55.685678 30006 raft_consensus.cc:504] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:55.686034 30006 leader_election.cc:288] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:55.686244 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d61d7974ed6144628d1cddb87122abdf" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:55.686343 22979 raft_consensus.cc:2454] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:55.686370 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d61d7974ed6144628d1cddb87122abdf" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:55.686501 25265 raft_consensus.cc:2454] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:55.686638 22520 leader_election.cc:302] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:55.686892 30006 raft_consensus.cc:2781] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:55.686956 30006 raft_consensus.cc:482] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:55.687002 30006 raft_consensus.cc:3037] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.687976 30006 raft_consensus.cc:504] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:55.688318 30006 leader_election.cc:288] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:55.688510 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d61d7974ed6144628d1cddb87122abdf" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:55.688592 22979 raft_consensus.cc:3037] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.688695 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d61d7974ed6144628d1cddb87122abdf" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:55.688803 25265 raft_consensus.cc:3037] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.689333 22979 raft_consensus.cc:2454] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:55.689533 22520 leader_election.cc:302] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:55.689677 30006 raft_consensus.cc:2781] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:55.689749 30006 raft_consensus.cc:686] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:55.689759 25265 raft_consensus.cc:2454] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:55.689877 30006 consensus_queue.cc:237] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:55.691623 22442 catalog_manager.cc:5484] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:55.764693 30783 raft_consensus.cc:482] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:55.764811 30783 raft_consensus.cc:504] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:55.765189 30783 leader_election.cc:288] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:55.765488 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "100d4b30eb344356943eb8cbe2c28bf2" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:55.765502 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "100d4b30eb344356943eb8cbe2c28bf2" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:55.765599 24870 raft_consensus.cc:2454] T 100d4b30eb344356943eb8cbe2c28bf2 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:55.765651 22979 raft_consensus.cc:2454] T 100d4b30eb344356943eb8cbe2c28bf2 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:55.765854 25068 leader_election.cc:302] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:55.766008 30783 raft_consensus.cc:2781] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:55.766060 30783 raft_consensus.cc:482] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:55.766132 30783 raft_consensus.cc:3037] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.766863 30783 raft_consensus.cc:504] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:55.767127 30783 leader_election.cc:288] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:55.767379 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "100d4b30eb344356943eb8cbe2c28bf2" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:55.767407 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "100d4b30eb344356943eb8cbe2c28bf2" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:55.767475 24870 raft_consensus.cc:3037] T 100d4b30eb344356943eb8cbe2c28bf2 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.767491 22979 raft_consensus.cc:3037] T 100d4b30eb344356943eb8cbe2c28bf2 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.768335 24870 raft_consensus.cc:2454] T 100d4b30eb344356943eb8cbe2c28bf2 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:55.768368 22979 raft_consensus.cc:2454] T 100d4b30eb344356943eb8cbe2c28bf2 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:55.768595 25068 leader_election.cc:302] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:55.768775 30783 raft_consensus.cc:2781] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:55.768863 30783 raft_consensus.cc:686] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:58:55.768980 30783 consensus_queue.cc:237] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:55.771028 22442 catalog_manager.cc:5484] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:55.790863 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:55.815506 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:55.819667 22979 raft_consensus.cc:1261] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:55.819667 25265 raft_consensus.cc:1261] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:55.820240 30708 consensus_queue.cc:1035] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:55.820523 30708 consensus_queue.cc:1035] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:55.822831 22436 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet d61d7974ed6144628d1cddb87122abdf (table rebalancer_dynamic_table [id=7cc1034a6792451a96febbce6db220ee]). Expected version 1 got 0
I20240502 07:58:55.823213 22444 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet d61d7974ed6144628d1cddb87122abdf (table rebalancer_dynamic_table [id=7cc1034a6792451a96febbce6db220ee]). Expected version 1 got 0
I20240502 07:58:55.823757 30847 tablet.cc:1722] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:55.824079 30847 tablet.cc:1620] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:58:55.825367 22444 catalog_manager.cc:5576] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451) does not have the latest schema for tablet d61d7974ed6144628d1cddb87122abdf (table rebalancer_dynamic_table [id=7cc1034a6792451a96febbce6db220ee]). Expected version 1 got 0
I20240502 07:58:55.826303 22442 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet d61d7974ed6144628d1cddb87122abdf (table rebalancer_dynamic_table [id=7cc1034a6792451a96febbce6db220ee]). Expected version 1 got 0
I20240502 07:58:55.826689 22444 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet d61d7974ed6144628d1cddb87122abdf (table rebalancer_dynamic_table [id=7cc1034a6792451a96febbce6db220ee]). Expected version 1 got 0
I20240502 07:58:55.827049 30850 tablet.cc:1722] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:55.827348 30850 tablet.cc:1620] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:55.827641 30849 tablet.cc:1722] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:55.827899 30849 tablet.cc:1620] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:58:55.830690 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=7cc1034a6792451a96febbce6db220ee] alter complete (version 1)
W20240502 07:58:55.861035 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:55.861726 30851 raft_consensus.cc:482] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:55.861827 30851 raft_consensus.cc:504] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:55.862218 30851 leader_election.cc:288] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:55.862551 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aedd9079c95746998d6b50a15fa96a9a" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:55.862557 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aedd9079c95746998d6b50a15fa96a9a" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:55.862686 24870 raft_consensus.cc:2454] T aedd9079c95746998d6b50a15fa96a9a P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:55.862684 25135 raft_consensus.cc:2454] T aedd9079c95746998d6b50a15fa96a9a P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:55.862927 22781 leader_election.cc:302] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:55.863229 30851 raft_consensus.cc:2781] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:55.863273 30851 raft_consensus.cc:482] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:55.863296 30851 raft_consensus.cc:3037] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
W20240502 07:58:55.863487 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:55.864208 30851 raft_consensus.cc:504] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:55.864506 30851 leader_election.cc:288] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:55.864814 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aedd9079c95746998d6b50a15fa96a9a" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:55.864766 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aedd9079c95746998d6b50a15fa96a9a" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:55.864918 25135 raft_consensus.cc:3037] T aedd9079c95746998d6b50a15fa96a9a P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.864917 24870 raft_consensus.cc:3037] T aedd9079c95746998d6b50a15fa96a9a P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:55.865852 24870 raft_consensus.cc:2454] T aedd9079c95746998d6b50a15fa96a9a P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:55.865852 25135 raft_consensus.cc:2454] T aedd9079c95746998d6b50a15fa96a9a P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:55.866140 22784 leader_election.cc:302] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:55.866339 30107 raft_consensus.cc:2781] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:55.866406 30107 raft_consensus.cc:686] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
W20240502 07:58:55.866477 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:55.866511 30107 consensus_queue.cc:237] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:55.868126 22444 catalog_manager.cc:5484] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:55.891268 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 07:58:55.927812 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:55.934175 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:55.939020 30847 tablet.cc:1722] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:55.939332 30847 tablet.cc:1620] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:58:55.939411 30850 tablet.cc:1722] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:55.939679 30850 tablet.cc:1620] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:55.940167 30849 tablet.cc:1722] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:55.940395 30849 tablet.cc:1620] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:58:55.944851 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=7cc1034a6792451a96febbce6db220ee] alter complete (version 2)
W20240502 07:58:56.024974 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 07:58:56.028705 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:56.047255 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:56.052029 30847 tablet.cc:1722] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:56.052486 30850 tablet.cc:1722] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:56.054199 30849 tablet.cc:1722] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:56.057700 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=7cc1034a6792451a96febbce6db220ee] alter complete (version 3)
I20240502 07:58:56.159817 22444 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:56.159947 22444 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:56.161662 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet d61d7974ed6144628d1cddb87122abdf
I20240502 07:58:56.162117 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet d61d7974ed6144628d1cddb87122abdf with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:56 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:58:56.162218 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet d61d7974ed6144628d1cddb87122abdf with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:56 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:56.162385 30889 tablet_replica.cc:330] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:58:56.162495 30890 tablet_replica.cc:330] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:56.162508 30889 raft_consensus.cc:2229] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:56.162621 30889 raft_consensus.cc:2258] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:56.162603 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet d61d7974ed6144628d1cddb87122abdf with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:56 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:56.162623 30890 raft_consensus.cc:2229] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:56.162885 30891 tablet_replica.cc:330] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:56.163014 30890 raft_consensus.cc:2258] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:56.163013 30891 raft_consensus.cc:2229] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:56.163127 30891 raft_consensus.cc:2258] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:56.163372 30889 ts_tablet_manager.cc:1875] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:56.163515 30890 ts_tablet_manager.cc:1875] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:56.163635 30891 ts_tablet_manager.cc:1875] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:56.165542 30890 ts_tablet_manager.cc:1888] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:56.165601 30890 log.cc:1196] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/d61d7974ed6144628d1cddb87122abdf
I20240502 07:58:56.165642 30889 ts_tablet_manager.cc:1888] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:56.165748 30889 log.cc:1196] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/d61d7974ed6144628d1cddb87122abdf
I20240502 07:58:56.165796 30891 ts_tablet_manager.cc:1888] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:56.165881 30891 log.cc:1196] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/d61d7974ed6144628d1cddb87122abdf
I20240502 07:58:56.165930 30890 ts_tablet_manager.cc:1909] T d61d7974ed6144628d1cddb87122abdf P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:56.166093 30889 ts_tablet_manager.cc:1909] T d61d7974ed6144628d1cddb87122abdf P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:58:56.166126 30891 ts_tablet_manager.cc:1909] T d61d7974ed6144628d1cddb87122abdf P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:58:56.166509 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet d61d7974ed6144628d1cddb87122abdf (table rebalancer_dynamic_table_new_name [id=7cc1034a6792451a96febbce6db220ee]) successfully deleted
I20240502 07:58:56.166656 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet d61d7974ed6144628d1cddb87122abdf (table rebalancer_dynamic_table_new_name [id=7cc1034a6792451a96febbce6db220ee]) successfully deleted
I20240502 07:58:56.166675 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet d61d7974ed6144628d1cddb87122abdf (table rebalancer_dynamic_table_new_name [id=7cc1034a6792451a96febbce6db220ee]) successfully deleted
I20240502 07:58:56.188383 30884 consensus_queue.cc:1035] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:56.191166 30884 consensus_queue.cc:1035] T 100d4b30eb344356943eb8cbe2c28bf2 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:56.203078 30685 consensus_queue.cc:1035] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:56.209452 30685 consensus_queue.cc:1035] T 1c2f5da964fe4ab2bbd71539fdf7a75e P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:56.263196 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:56.267951 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 42ea2f9d0ec74a27a8cee93eea9b96d8 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=f31f292bcb454519b16b5b97ef652d02]), partition=
I20240502 07:58:56.268075 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 42ea2f9d0ec74a27a8cee93eea9b96d8 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=f31f292bcb454519b16b5b97ef652d02]), partition=
I20240502 07:58:56.268153 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 42ea2f9d0ec74a27a8cee93eea9b96d8 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=f31f292bcb454519b16b5b97ef652d02]), partition=
I20240502 07:58:56.268193 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 42ea2f9d0ec74a27a8cee93eea9b96d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:56.268301 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 42ea2f9d0ec74a27a8cee93eea9b96d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:56.268334 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 42ea2f9d0ec74a27a8cee93eea9b96d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:56.270303 30930 tablet_bootstrap.cc:492] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:56.270478 30929 tablet_bootstrap.cc:492] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:56.271421 30929 tablet_bootstrap.cc:654] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:56.271519 30930 tablet_bootstrap.cc:654] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:56.272116 30931 tablet_bootstrap.cc:492] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:56.272862 30929 tablet_bootstrap.cc:492] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:56.272948 30929 ts_tablet_manager.cc:1367] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:56.272964 30931 tablet_bootstrap.cc:654] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:56.273274 30929 raft_consensus.cc:348] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:56.273393 30929 raft_consensus.cc:374] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:56.273442 30929 raft_consensus.cc:729] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:56.273597 30929 consensus_queue.cc:260] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:56.273816 30930 tablet_bootstrap.cc:492] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:56.273836 30931 tablet_bootstrap.cc:492] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:56.273891 30930 ts_tablet_manager.cc:1367] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:56.273901 30931 ts_tablet_manager.cc:1367] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:56.273944 30929 ts_tablet_manager.cc:1398] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:56.274343 30930 raft_consensus.cc:348] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:56.274441 30930 raft_consensus.cc:374] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:56.274469 30930 raft_consensus.cc:729] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:56.274570 30930 consensus_queue.cc:260] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:56.274874 30930 ts_tablet_manager.cc:1398] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:56.274358 30931 raft_consensus.cc:348] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:56.275682 30931 raft_consensus.cc:374] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:56.275839 30931 raft_consensus.cc:729] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:56.275979 30931 consensus_queue.cc:260] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:56.276201 30931 ts_tablet_manager.cc:1398] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:56.282027 30455 consensus_queue.cc:1035] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:56.284988 30107 consensus_queue.cc:1035] T aedd9079c95746998d6b50a15fa96a9a P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:56.308636 30708 raft_consensus.cc:482] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:56.308755 30708 raft_consensus.cc:504] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:56.309126 30708 leader_election.cc:288] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:56.309392 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "42ea2f9d0ec74a27a8cee93eea9b96d8" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:56.309422 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "42ea2f9d0ec74a27a8cee93eea9b96d8" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:56.309509 25265 raft_consensus.cc:2454] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:56.309509 25135 raft_consensus.cc:2454] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:56.309727 22522 leader_election.cc:302] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:56.309891 30708 raft_consensus.cc:2781] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:56.309947 30708 raft_consensus.cc:482] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:56.309988 30708 raft_consensus.cc:3037] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:56.310710 30708 raft_consensus.cc:504] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:56.310992 30708 leader_election.cc:288] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:56.311215 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "42ea2f9d0ec74a27a8cee93eea9b96d8" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:56.311221 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "42ea2f9d0ec74a27a8cee93eea9b96d8" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:56.311293 25135 raft_consensus.cc:3037] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:56.311329 25265 raft_consensus.cc:3037] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:56.311949 25135 raft_consensus.cc:2454] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:56.312096 25265 raft_consensus.cc:2454] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:56.312176 22522 leader_election.cc:302] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:56.312408 30708 raft_consensus.cc:2781] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:56.312500 30708 raft_consensus.cc:686] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:56.312616 30708 consensus_queue.cc:237] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:56.314342 22444 catalog_manager.cc:5484] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:56.361327 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:56.385246 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_36"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:56.390648 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 5303b025ea014dbeb652cab8f75fb5eb (DEFAULT_TABLE table=rebalancer_extra_table_36 [id=40a413268a6147f7baf5045ef76261df]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:56.390667 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 5303b025ea014dbeb652cab8f75fb5eb (DEFAULT_TABLE table=rebalancer_extra_table_36 [id=40a413268a6147f7baf5045ef76261df]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:56.390977 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5303b025ea014dbeb652cab8f75fb5eb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:56.390980 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5303b025ea014dbeb652cab8f75fb5eb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:56.391449 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 5da863a6125644539d8c0260bc34c8cf (DEFAULT_TABLE table=rebalancer_extra_table_36 [id=40a413268a6147f7baf5045ef76261df]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:56.391671 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5da863a6125644539d8c0260bc34c8cf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:56.392812 30940 tablet_bootstrap.cc:492] T 5303b025ea014dbeb652cab8f75fb5eb P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:56.393004 22566 tablet_service.cc:1467] Processing CreateTablet for tablet fc93c3228f1b4162864dced4a66be7a5 (DEFAULT_TABLE table=rebalancer_extra_table_36 [id=40a413268a6147f7baf5045ef76261df]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:56.393229 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc93c3228f1b4162864dced4a66be7a5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:56.393647 30940 tablet_bootstrap.cc:654] T 5303b025ea014dbeb652cab8f75fb5eb P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:56.393657 30942 tablet_bootstrap.cc:492] T 5da863a6125644539d8c0260bc34c8cf P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:56.393838 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 5da863a6125644539d8c0260bc34c8cf (DEFAULT_TABLE table=rebalancer_extra_table_36 [id=40a413268a6147f7baf5045ef76261df]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:56.394088 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5da863a6125644539d8c0260bc34c8cf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:56.394515 30942 tablet_bootstrap.cc:654] T 5da863a6125644539d8c0260bc34c8cf P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:56.395009 30929 tablet_bootstrap.cc:492] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:56.395184 25235 tablet_service.cc:1467] Processing CreateTablet for tablet fc93c3228f1b4162864dced4a66be7a5 (DEFAULT_TABLE table=rebalancer_extra_table_36 [id=40a413268a6147f7baf5045ef76261df]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:56.395419 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc93c3228f1b4162864dced4a66be7a5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:56.395836 30929 tablet_bootstrap.cc:654] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:56.395879 30931 tablet_bootstrap.cc:492] T 5da863a6125644539d8c0260bc34c8cf P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:56.396306 30941 tablet_bootstrap.cc:492] T 5303b025ea014dbeb652cab8f75fb5eb P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:56.396916 30931 tablet_bootstrap.cc:654] T 5da863a6125644539d8c0260bc34c8cf P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:56.397162 30929 tablet_bootstrap.cc:492] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:56.397234 30929 ts_tablet_manager.cc:1367] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:56.397281 30940 tablet_bootstrap.cc:492] T 5303b025ea014dbeb652cab8f75fb5eb P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:56.397361 30940 ts_tablet_manager.cc:1367] T 5303b025ea014dbeb652cab8f75fb5eb P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:58:56.397645 30929 raft_consensus.cc:348] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.397660 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 5303b025ea014dbeb652cab8f75fb5eb (DEFAULT_TABLE table=rebalancer_extra_table_36 [id=40a413268a6147f7baf5045ef76261df]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:56.397739 30929 raft_consensus.cc:374] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:56.397773 30929 raft_consensus.cc:729] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:56.397769 30940 raft_consensus.cc:348] T 5303b025ea014dbeb652cab8f75fb5eb P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.397868 30940 raft_consensus.cc:374] T 5303b025ea014dbeb652cab8f75fb5eb P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:56.397894 30940 raft_consensus.cc:729] T 5303b025ea014dbeb652cab8f75fb5eb P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:56.397903 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5303b025ea014dbeb652cab8f75fb5eb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:56.398001 30940 consensus_queue.cc:260] T 5303b025ea014dbeb652cab8f75fb5eb P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.398245 30940 ts_tablet_manager.cc:1398] T 5303b025ea014dbeb652cab8f75fb5eb P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:56.399075 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 5da863a6125644539d8c0260bc34c8cf (DEFAULT_TABLE table=rebalancer_extra_table_36 [id=40a413268a6147f7baf5045ef76261df]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:56.399271 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5da863a6125644539d8c0260bc34c8cf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:56.399472 30941 tablet_bootstrap.cc:654] T 5303b025ea014dbeb652cab8f75fb5eb P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
W20240502 07:58:56.399682 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:56.400475 30931 tablet_bootstrap.cc:492] T 5da863a6125644539d8c0260bc34c8cf P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:56.400600 30931 ts_tablet_manager.cc:1367] T 5da863a6125644539d8c0260bc34c8cf P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:58:56.400918 30942 tablet_bootstrap.cc:492] T 5da863a6125644539d8c0260bc34c8cf P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:56.400990 30942 ts_tablet_manager.cc:1367] T 5da863a6125644539d8c0260bc34c8cf P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 07:58:56.401042 30931 raft_consensus.cc:348] T 5da863a6125644539d8c0260bc34c8cf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.401129 30931 raft_consensus.cc:374] T 5da863a6125644539d8c0260bc34c8cf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:56.401124 30930 tablet_bootstrap.cc:492] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:56.401157 30931 raft_consensus.cc:729] T 5da863a6125644539d8c0260bc34c8cf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:56.401257 30931 consensus_queue.cc:260] T 5da863a6125644539d8c0260bc34c8cf P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.401379 30942 raft_consensus.cc:348] T 5da863a6125644539d8c0260bc34c8cf P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.401460 30942 raft_consensus.cc:374] T 5da863a6125644539d8c0260bc34c8cf P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:56.401487 30942 raft_consensus.cc:729] T 5da863a6125644539d8c0260bc34c8cf P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:56.401523 30931 ts_tablet_manager.cc:1398] T 5da863a6125644539d8c0260bc34c8cf P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:56.401661 30931 tablet_bootstrap.cc:492] T fc93c3228f1b4162864dced4a66be7a5 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:56.401641 30942 consensus_queue.cc:260] T 5da863a6125644539d8c0260bc34c8cf P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.401875 30942 ts_tablet_manager.cc:1398] T 5da863a6125644539d8c0260bc34c8cf P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:56.401985 30930 tablet_bootstrap.cc:654] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:56.402294 30941 tablet_bootstrap.cc:492] T 5303b025ea014dbeb652cab8f75fb5eb P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:56.402362 30941 ts_tablet_manager.cc:1367] T 5303b025ea014dbeb652cab8f75fb5eb P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:58:56.402755 30941 raft_consensus.cc:348] T 5303b025ea014dbeb652cab8f75fb5eb P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.402838 30941 raft_consensus.cc:374] T 5303b025ea014dbeb652cab8f75fb5eb P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:56.402863 30941 raft_consensus.cc:729] T 5303b025ea014dbeb652cab8f75fb5eb P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:56.402961 30941 consensus_queue.cc:260] T 5303b025ea014dbeb652cab8f75fb5eb P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.403187 30941 ts_tablet_manager.cc:1398] T 5303b025ea014dbeb652cab8f75fb5eb P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:56.403574 30930 tablet_bootstrap.cc:492] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:56.403651 30930 ts_tablet_manager.cc:1367] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:56.404067 30930 raft_consensus.cc:348] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.404171 30930 raft_consensus.cc:374] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:56.404201 30930 raft_consensus.cc:729] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:56.404237 30931 tablet_bootstrap.cc:654] T fc93c3228f1b4162864dced4a66be7a5 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:56.404316 30930 consensus_queue.cc:260] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.404614 30930 ts_tablet_manager.cc:1398] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:56.404865 30930 tablet_bootstrap.cc:492] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:56.405371 25114 tablet_service.cc:1467] Processing CreateTablet for tablet fc93c3228f1b4162864dced4a66be7a5 (DEFAULT_TABLE table=rebalancer_extra_table_36 [id=40a413268a6147f7baf5045ef76261df]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:56.405714 30929 consensus_queue.cc:260] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.405979 30929 ts_tablet_manager.cc:1398] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.009s user 0.001s sys 0.000s
I20240502 07:58:56.407162 25114 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc93c3228f1b4162864dced4a66be7a5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:56.408573 30930 tablet_bootstrap.cc:654] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:56.409487 30930 tablet_bootstrap.cc:492] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:56.409590 30930 ts_tablet_manager.cc:1367] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.002s
I20240502 07:58:56.410032 30930 raft_consensus.cc:348] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.410141 30930 raft_consensus.cc:374] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:56.410187 30930 raft_consensus.cc:729] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:56.410292 30930 consensus_queue.cc:260] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.410548 30930 ts_tablet_manager.cc:1398] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:56.410708 30930 tablet_bootstrap.cc:492] T fc93c3228f1b4162864dced4a66be7a5 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:56.411268 30931 tablet_bootstrap.cc:492] T fc93c3228f1b4162864dced4a66be7a5 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:56.411367 30931 ts_tablet_manager.cc:1367] T fc93c3228f1b4162864dced4a66be7a5 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.000s
I20240502 07:58:56.411623 30930 tablet_bootstrap.cc:654] T fc93c3228f1b4162864dced4a66be7a5 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:56.413087 30930 tablet_bootstrap.cc:492] T fc93c3228f1b4162864dced4a66be7a5 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:56.413197 30930 ts_tablet_manager.cc:1367] T fc93c3228f1b4162864dced4a66be7a5 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240502 07:58:56.413484 30931 raft_consensus.cc:348] T fc93c3228f1b4162864dced4a66be7a5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.413563 30930 raft_consensus.cc:348] T fc93c3228f1b4162864dced4a66be7a5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.413640 30931 raft_consensus.cc:374] T fc93c3228f1b4162864dced4a66be7a5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:56.413686 30930 raft_consensus.cc:374] T fc93c3228f1b4162864dced4a66be7a5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:56.413722 30930 raft_consensus.cc:729] T fc93c3228f1b4162864dced4a66be7a5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:56.413749 30931 raft_consensus.cc:729] T fc93c3228f1b4162864dced4a66be7a5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:56.413854 30930 consensus_queue.cc:260] T fc93c3228f1b4162864dced4a66be7a5 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.413990 30931 consensus_queue.cc:260] T fc93c3228f1b4162864dced4a66be7a5 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.414125 30930 ts_tablet_manager.cc:1398] T fc93c3228f1b4162864dced4a66be7a5 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:56.414414 30931 ts_tablet_manager.cc:1398] T fc93c3228f1b4162864dced4a66be7a5 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:56.420182 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:56.423946 25265 raft_consensus.cc:1261] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:56.424036 25135 raft_consensus.cc:1261] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:56.424427 30848 consensus_queue.cc:1035] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:56.424557 30708 consensus_queue.cc:1035] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:56.429309 22442 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 42ea2f9d0ec74a27a8cee93eea9b96d8 (table rebalancer_dynamic_table [id=f31f292bcb454519b16b5b97ef652d02]). Expected version 1 got 0
I20240502 07:58:56.430241 30843 mvcc.cc:205] Tried to move back new op lower bound from 7023152072389767168 to 7023152071937794048. Current Snapshot: MvccSnapshot[applied={T|T < 7023152072389767168}]
I20240502 07:58:56.430285 22440 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet 42ea2f9d0ec74a27a8cee93eea9b96d8 (table rebalancer_dynamic_table [id=f31f292bcb454519b16b5b97ef652d02]). Expected version 1 got 0
I20240502 07:58:56.430325 30847 tablet.cc:1722] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:56.430583 30847 tablet.cc:1620] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:58:56.434548 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=f31f292bcb454519b16b5b97ef652d02] alter complete (version 1)
W20240502 07:58:56.436990 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:56.437686 22444 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 42ea2f9d0ec74a27a8cee93eea9b96d8 (table rebalancer_dynamic_table [id=f31f292bcb454519b16b5b97ef652d02]). Expected version 1 got 0
I20240502 07:58:56.439002 30960 tablet.cc:1722] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:56.439291 30960 tablet.cc:1620] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:56.440999 30574 mvcc.cc:205] Tried to move back new op lower bound from 7023152072389767168 to 7023152071937794048. Current Snapshot: MvccSnapshot[applied={T|T < 7023152072389767168}]
I20240502 07:58:56.441176 30850 tablet.cc:1722] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:56.441438 30850 tablet.cc:1620] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:56.442898 22444 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet 42ea2f9d0ec74a27a8cee93eea9b96d8 (table rebalancer_dynamic_table [id=f31f292bcb454519b16b5b97ef652d02]). Expected version 1 got 0
I20240502 07:58:56.488829 30006 raft_consensus.cc:482] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:56.488936 30006 raft_consensus.cc:504] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.489244 30006 leader_election.cc:288] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:56.489516 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc93c3228f1b4162864dced4a66be7a5" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:56.489514 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc93c3228f1b4162864dced4a66be7a5" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:56.489630 25133 raft_consensus.cc:2454] T fc93c3228f1b4162864dced4a66be7a5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:56.489635 25265 raft_consensus.cc:2454] T fc93c3228f1b4162864dced4a66be7a5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:56.489899 22522 leader_election.cc:302] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:56.490047 30006 raft_consensus.cc:2781] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:56.490104 30006 raft_consensus.cc:482] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:56.490142 30006 raft_consensus.cc:3037] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:56.491070 30006 raft_consensus.cc:504] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.491369 30006 leader_election.cc:288] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:56.491655 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc93c3228f1b4162864dced4a66be7a5" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:56.491763 25133 raft_consensus.cc:3037] T fc93c3228f1b4162864dced4a66be7a5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
W20240502 07:58:56.491842 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:56.492566 25133 raft_consensus.cc:2454] T fc93c3228f1b4162864dced4a66be7a5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:56.492825 22522 leader_election.cc:302] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:56.492839 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc93c3228f1b4162864dced4a66be7a5" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:56.492955 30006 raft_consensus.cc:2781] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:56.492975 25265 raft_consensus.cc:3037] T fc93c3228f1b4162864dced4a66be7a5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:56.493023 30006 raft_consensus.cc:686] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:56.493129 30006 consensus_queue.cc:237] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.493891 25265 raft_consensus.cc:2454] T fc93c3228f1b4162864dced4a66be7a5 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:56.495543 22442 catalog_manager.cc:5484] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:56.509022 30884 raft_consensus.cc:482] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:56.509114 30884 raft_consensus.cc:504] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.509198 30535 raft_consensus.cc:482] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:56.509253 30535 raft_consensus.cc:504] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.509465 30884 leader_election.cc:288] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:56.509498 30535 leader_election.cc:288] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:56.509725 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5da863a6125644539d8c0260bc34c8cf" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:56.509727 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5303b025ea014dbeb652cab8f75fb5eb" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:56.509835 25265 raft_consensus.cc:2454] T 5da863a6125644539d8c0260bc34c8cf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:56.509842 22847 raft_consensus.cc:2454] T 5303b025ea014dbeb652cab8f75fb5eb P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:56.509905 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5303b025ea014dbeb652cab8f75fb5eb" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:56.510023 22716 raft_consensus.cc:2454] T 5303b025ea014dbeb652cab8f75fb5eb P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:56.510046 25070 leader_election.cc:302] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:56.510201 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5da863a6125644539d8c0260bc34c8cf" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:56.510291 25068 leader_election.cc:302] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:56.510327 22978 raft_consensus.cc:2454] T 5da863a6125644539d8c0260bc34c8cf P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:56.510426 30884 raft_consensus.cc:2781] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:56.510471 30535 raft_consensus.cc:2781] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:56.510481 30884 raft_consensus.cc:482] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:56.510533 30535 raft_consensus.cc:482] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:56.510553 30884 raft_consensus.cc:3037] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:56.510609 30535 raft_consensus.cc:3037] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:56.511601 30535 raft_consensus.cc:504] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.511646 30884 raft_consensus.cc:504] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.511929 30535 leader_election.cc:288] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:56.512178 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5da863a6125644539d8c0260bc34c8cf" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:56.512284 25265 raft_consensus.cc:3037] T 5da863a6125644539d8c0260bc34c8cf P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:56.512301 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5da863a6125644539d8c0260bc34c8cf" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:56.512389 22978 raft_consensus.cc:3037] T 5da863a6125644539d8c0260bc34c8cf P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:56.512713 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5303b025ea014dbeb652cab8f75fb5eb" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:56.512812 22847 raft_consensus.cc:3037] T 5303b025ea014dbeb652cab8f75fb5eb P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:56.513159 22978 raft_consensus.cc:2454] T 5da863a6125644539d8c0260bc34c8cf P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:56.513383 25069 leader_election.cc:302] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 4654231dfe994c1a9d2e37d9c3d75699; no voters:
I20240502 07:58:56.513516 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5303b025ea014dbeb652cab8f75fb5eb" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:56.513607 22716 raft_consensus.cc:3037] T 5303b025ea014dbeb652cab8f75fb5eb P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:56.513681 22847 raft_consensus.cc:2454] T 5303b025ea014dbeb652cab8f75fb5eb P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:56.513936 25071 leader_election.cc:302] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:56.514061 30783 raft_consensus.cc:2781] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:56.514215 30783 raft_consensus.cc:686] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:58:56.514344 30783 consensus_queue.cc:237] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.514500 22716 raft_consensus.cc:2454] T 5303b025ea014dbeb652cab8f75fb5eb P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:56.514632 25265 raft_consensus.cc:2454] T 5da863a6125644539d8c0260bc34c8cf P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:56.514932 30535 raft_consensus.cc:2781] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:56.515008 30535 raft_consensus.cc:686] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:58:56.515108 30535 consensus_queue.cc:237] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:56.511929 30884 leader_election.cc:288] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:56.516629 22442 catalog_manager.cc:5484] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:56.519942 22442 catalog_manager.cc:5484] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:56.540052 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:56.544505 30847 tablet.cc:1722] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:56.544854 30847 tablet.cc:1620] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:58:56.544837 30960 tablet.cc:1722] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:56.545114 30960 tablet.cc:1620] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:56.545637 30850 tablet.cc:1722] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:56.545887 30850 tablet.cc:1620] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:56.549505 22442 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=f31f292bcb454519b16b5b97ef652d02] alter complete (version 2)
W20240502 07:58:56.610229 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): CreateTablet RPC failed for tablet f1862e3480644235adaf4a58f2c45bd9: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:58:56.616485 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:56.652545 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:56.657344 30847 tablet.cc:1722] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:56.657727 30960 tablet.cc:1722] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:56.659922 30850 tablet.cc:1722] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:56.662645 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=f31f292bcb454519b16b5b97ef652d02] alter complete (version 3)
I20240502 07:58:56.766252 22440 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:56.766397 22440 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:56.768241 22440 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 42ea2f9d0ec74a27a8cee93eea9b96d8
I20240502 07:58:56.768837 25114 tablet_service.cc:1514] Processing DeleteTablet for tablet 42ea2f9d0ec74a27a8cee93eea9b96d8 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:56 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:56.768833 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 42ea2f9d0ec74a27a8cee93eea9b96d8 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:56 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:56.769188 31024 tablet_replica.cc:330] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:56.769260 31023 tablet_replica.cc:330] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:56.769340 31024 raft_consensus.cc:2229] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:56.769434 31023 raft_consensus.cc:2229] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:56.769466 31024 raft_consensus.cc:2258] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:56.769634 31023 raft_consensus.cc:2258] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:56.769909 25235 tablet_service.cc:1514] Processing DeleteTablet for tablet 42ea2f9d0ec74a27a8cee93eea9b96d8 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:56 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:56.770180 31025 tablet_replica.cc:330] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:56.770319 31025 raft_consensus.cc:2229] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:56.770406 31024 ts_tablet_manager.cc:1875] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:56.770466 31025 raft_consensus.cc:2258] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:56.770963 31023 ts_tablet_manager.cc:1875] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:56.771036 31025 ts_tablet_manager.cc:1875] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:56.772289 31024 ts_tablet_manager.cc:1888] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:56.772352 31024 log.cc:1196] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/42ea2f9d0ec74a27a8cee93eea9b96d8
I20240502 07:58:56.772687 31024 ts_tablet_manager.cc:1909] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:56.773031 31023 ts_tablet_manager.cc:1888] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:56.773032 31025 ts_tablet_manager.cc:1888] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:56.773100 31023 log.cc:1196] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/42ea2f9d0ec74a27a8cee93eea9b96d8
I20240502 07:58:56.773100 31025 log.cc:1196] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/42ea2f9d0ec74a27a8cee93eea9b96d8
I20240502 07:58:56.773298 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet 42ea2f9d0ec74a27a8cee93eea9b96d8 (table rebalancer_dynamic_table_new_name [id=f31f292bcb454519b16b5b97ef652d02]) successfully deleted
I20240502 07:58:56.773350 31025 ts_tablet_manager.cc:1909] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:58:56.773401 31023 ts_tablet_manager.cc:1909] T 42ea2f9d0ec74a27a8cee93eea9b96d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:56.773965 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet 42ea2f9d0ec74a27a8cee93eea9b96d8 (table rebalancer_dynamic_table_new_name [id=f31f292bcb454519b16b5b97ef652d02]) successfully deleted
I20240502 07:58:56.773991 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 42ea2f9d0ec74a27a8cee93eea9b96d8 (table rebalancer_dynamic_table_new_name [id=f31f292bcb454519b16b5b97ef652d02]) successfully deleted
I20240502 07:58:56.870357 22440 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:56.874951 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 4241ac2fc3844670b048965ff580e06b (DEFAULT_TABLE table=rebalancer_dynamic_table [id=5bf11aec70844146a3363d737c4444bf]), partition=
I20240502 07:58:56.874951 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 4241ac2fc3844670b048965ff580e06b (DEFAULT_TABLE table=rebalancer_dynamic_table [id=5bf11aec70844146a3363d737c4444bf]), partition=
I20240502 07:58:56.875217 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4241ac2fc3844670b048965ff580e06b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:56.875217 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4241ac2fc3844670b048965ff580e06b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:56.877224 30929 tablet_bootstrap.cc:492] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:56.877305 31028 tablet_bootstrap.cc:492] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:56.877473 25114 tablet_service.cc:1467] Processing CreateTablet for tablet 4241ac2fc3844670b048965ff580e06b (DEFAULT_TABLE table=rebalancer_dynamic_table [id=5bf11aec70844146a3363d737c4444bf]), partition=
I20240502 07:58:56.877676 25114 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4241ac2fc3844670b048965ff580e06b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:56.878198 30929 tablet_bootstrap.cc:654] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:56.878263 31028 tablet_bootstrap.cc:654] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:56.879784 30929 tablet_bootstrap.cc:492] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:56.879859 30929 ts_tablet_manager.cc:1367] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:56.879870 31028 tablet_bootstrap.cc:492] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:56.879930 30930 tablet_bootstrap.cc:492] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:56.879942 31028 ts_tablet_manager.cc:1367] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:56.880245 30929 raft_consensus.cc:348] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:56.880323 30929 raft_consensus.cc:374] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:56.880342 30929 raft_consensus.cc:729] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:56.880419 31028 raft_consensus.cc:348] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:56.880435 30929 consensus_queue.cc:260] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:56.880517 31028 raft_consensus.cc:374] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:56.880543 31028 raft_consensus.cc:729] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:56.880759 30930 tablet_bootstrap.cc:654] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:56.880698 31028 consensus_queue.cc:260] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:56.881623 30929 ts_tablet_manager.cc:1398] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:56.882450 31028 ts_tablet_manager.cc:1398] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:56.884198 30930 tablet_bootstrap.cc:492] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:56.884281 30930 ts_tablet_manager.cc:1367] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.001s
I20240502 07:58:56.884646 30930 raft_consensus.cc:348] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:56.884778 30930 raft_consensus.cc:374] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:56.884817 30930 raft_consensus.cc:729] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:56.884950 30930 consensus_queue.cc:260] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:56.885201 30930 ts_tablet_manager.cc:1398] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:56.903671 30708 consensus_queue.cc:1035] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:56.907486 30708 consensus_queue.cc:1035] T fc93c3228f1b4162864dced4a66be7a5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:56.923564 30987 consensus_queue.cc:1035] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:56.929016 30987 consensus_queue.cc:1035] T 5da863a6125644539d8c0260bc34c8cf P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:58:56.938498 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:57.026737 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_38"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:57.032486 22695 tablet_service.cc:1467] Processing CreateTablet for tablet c9cc4e72c4f242eab0620c1869bf4c5f (DEFAULT_TABLE table=rebalancer_extra_table_38 [id=b71f04f23d4a47dfbc907f699f01c73a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.032828 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9cc4e72c4f242eab0620c1869bf4c5f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.033111 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 0a28754835d5412ab4ec96600bd8d2a4 (DEFAULT_TABLE table=rebalancer_extra_table_38 [id=b71f04f23d4a47dfbc907f699f01c73a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.033375 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0a28754835d5412ab4ec96600bd8d2a4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.033643 25114 tablet_service.cc:1467] Processing CreateTablet for tablet c9cc4e72c4f242eab0620c1869bf4c5f (DEFAULT_TABLE table=rebalancer_extra_table_38 [id=b71f04f23d4a47dfbc907f699f01c73a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.033900 25114 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9cc4e72c4f242eab0620c1869bf4c5f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.035147 31068 tablet_bootstrap.cc:492] T c9cc4e72c4f242eab0620c1869bf4c5f P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:57.035310 24852 tablet_service.cc:1467] Processing CreateTablet for tablet fbcaa8ec241341a2b56f1e6f7e14f430 (DEFAULT_TABLE table=rebalancer_extra_table_38 [id=b71f04f23d4a47dfbc907f699f01c73a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.035506 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fbcaa8ec241341a2b56f1e6f7e14f430. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.035485 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 0a28754835d5412ab4ec96600bd8d2a4 (DEFAULT_TABLE table=rebalancer_extra_table_38 [id=b71f04f23d4a47dfbc907f699f01c73a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.035742 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0a28754835d5412ab4ec96600bd8d2a4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.035965 30930 tablet_bootstrap.cc:492] T c9cc4e72c4f242eab0620c1869bf4c5f P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:57.036101 31068 tablet_bootstrap.cc:654] T c9cc4e72c4f242eab0620c1869bf4c5f P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.036900 30930 tablet_bootstrap.cc:654] T c9cc4e72c4f242eab0620c1869bf4c5f P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.036926 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 0a28754835d5412ab4ec96600bd8d2a4 (DEFAULT_TABLE table=rebalancer_extra_table_38 [id=b71f04f23d4a47dfbc907f699f01c73a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.037122 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0a28754835d5412ab4ec96600bd8d2a4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.037906 31071 tablet_bootstrap.cc:492] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:57.038014 22827 tablet_service.cc:1467] Processing CreateTablet for tablet fbcaa8ec241341a2b56f1e6f7e14f430 (DEFAULT_TABLE table=rebalancer_extra_table_38 [id=b71f04f23d4a47dfbc907f699f01c73a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.038235 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fbcaa8ec241341a2b56f1e6f7e14f430. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.038736 31071 tablet_bootstrap.cc:654] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.039918 30930 tablet_bootstrap.cc:492] T c9cc4e72c4f242eab0620c1869bf4c5f P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:57.039986 30930 ts_tablet_manager.cc:1367] T c9cc4e72c4f242eab0620c1869bf4c5f P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.001s
I20240502 07:58:57.040099 31028 tablet_bootstrap.cc:492] T 0a28754835d5412ab4ec96600bd8d2a4 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:57.040381 30930 raft_consensus.cc:348] T c9cc4e72c4f242eab0620c1869bf4c5f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.040488 30930 raft_consensus.cc:374] T c9cc4e72c4f242eab0620c1869bf4c5f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.040515 30930 raft_consensus.cc:729] T c9cc4e72c4f242eab0620c1869bf4c5f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.040616 30930 consensus_queue.cc:260] T c9cc4e72c4f242eab0620c1869bf4c5f P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.040849 30884 raft_consensus.cc:482] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:57.040907 30930 ts_tablet_manager.cc:1398] T c9cc4e72c4f242eab0620c1869bf4c5f P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:57.040956 31028 tablet_bootstrap.cc:654] T 0a28754835d5412ab4ec96600bd8d2a4 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.040915 30884 raft_consensus.cc:504] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:57.041404 30884 leader_election.cc:288] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:57.041679 31072 tablet_bootstrap.cc:492] T 0a28754835d5412ab4ec96600bd8d2a4 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:57.041661 22957 tablet_service.cc:1467] Processing CreateTablet for tablet c9cc4e72c4f242eab0620c1869bf4c5f (DEFAULT_TABLE table=rebalancer_extra_table_38 [id=b71f04f23d4a47dfbc907f699f01c73a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.041890 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9cc4e72c4f242eab0620c1869bf4c5f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.042352 31072 tablet_bootstrap.cc:654] T 0a28754835d5412ab4ec96600bd8d2a4 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.043576 31028 tablet_bootstrap.cc:492] T 0a28754835d5412ab4ec96600bd8d2a4 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:57.043633 31028 ts_tablet_manager.cc:1367] T 0a28754835d5412ab4ec96600bd8d2a4 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:57.043885 31076 tablet_bootstrap.cc:492] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:57.044010 31028 raft_consensus.cc:348] T 0a28754835d5412ab4ec96600bd8d2a4 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:57.044096 31028 raft_consensus.cc:374] T 0a28754835d5412ab4ec96600bd8d2a4 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.044126 31028 raft_consensus.cc:729] T 0a28754835d5412ab4ec96600bd8d2a4 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.044262 31028 consensus_queue.cc:260] T 0a28754835d5412ab4ec96600bd8d2a4 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:57.044504 31028 ts_tablet_manager.cc:1398] T 0a28754835d5412ab4ec96600bd8d2a4 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:57.044698 31028 tablet_bootstrap.cc:492] T fbcaa8ec241341a2b56f1e6f7e14f430 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:57.045626 31076 tablet_bootstrap.cc:654] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.046171 22566 tablet_service.cc:1467] Processing CreateTablet for tablet fbcaa8ec241341a2b56f1e6f7e14f430 (DEFAULT_TABLE table=rebalancer_extra_table_38 [id=b71f04f23d4a47dfbc907f699f01c73a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.046449 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fbcaa8ec241341a2b56f1e6f7e14f430. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.046615 31076 tablet_bootstrap.cc:492] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:57.046695 31076 ts_tablet_manager.cc:1367] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:57.046695 31028 tablet_bootstrap.cc:654] T fbcaa8ec241341a2b56f1e6f7e14f430 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.047103 31076 raft_consensus.cc:348] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.047195 31076 raft_consensus.cc:374] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.047165 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4241ac2fc3844670b048965ff580e06b" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:57.047228 31076 raft_consensus.cc:729] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.047276 22586 raft_consensus.cc:2454] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:57.047331 31076 consensus_queue.cc:260] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.047593 31076 ts_tablet_manager.cc:1398] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:57.048424 30929 tablet_bootstrap.cc:492] T fbcaa8ec241341a2b56f1e6f7e14f430 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:57.048617 25071 leader_election.cc:302] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:57.048929 30987 raft_consensus.cc:2781] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:57.048976 30987 raft_consensus.cc:482] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:57.049008 30987 raft_consensus.cc:3037] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.049265 30929 tablet_bootstrap.cc:654] T fbcaa8ec241341a2b56f1e6f7e14f430 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.049939 30987 raft_consensus.cc:504] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:57.050267 30987 leader_election.cc:288] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:57.050541 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4241ac2fc3844670b048965ff580e06b" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:57.050684 24870 raft_consensus.cc:2454] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:58:57.050704 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4241ac2fc3844670b048965ff580e06b" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:57.050801 24872 raft_consensus.cc:3037] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.051124 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4241ac2fc3844670b048965ff580e06b" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:57.051215 22586 raft_consensus.cc:3037] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.052064 30929 tablet_bootstrap.cc:492] T fbcaa8ec241341a2b56f1e6f7e14f430 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:57.052129 22586 raft_consensus.cc:2454] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:57.052183 30929 ts_tablet_manager.cc:1367] T fbcaa8ec241341a2b56f1e6f7e14f430 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:57.052400 25071 leader_election.cc:302] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:57.052546 30987 raft_consensus.cc:2781] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:57.052616 30987 raft_consensus.cc:686] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:58:57.052609 30929 raft_consensus.cc:348] T fbcaa8ec241341a2b56f1e6f7e14f430 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:57.052712 31068 tablet_bootstrap.cc:492] T c9cc4e72c4f242eab0620c1869bf4c5f P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:57.052723 30929 raft_consensus.cc:374] T fbcaa8ec241341a2b56f1e6f7e14f430 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.052754 30929 raft_consensus.cc:729] T fbcaa8ec241341a2b56f1e6f7e14f430 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.052744 30987 consensus_queue.cc:237] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:57.052786 31072 tablet_bootstrap.cc:492] T 0a28754835d5412ab4ec96600bd8d2a4 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:57.052793 31068 ts_tablet_manager.cc:1367] T c9cc4e72c4f242eab0620c1869bf4c5f P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.018s user 0.002s sys 0.000s
I20240502 07:58:57.052863 31072 ts_tablet_manager.cc:1367] T 0a28754835d5412ab4ec96600bd8d2a4 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.000s
I20240502 07:58:57.052866 30929 consensus_queue.cc:260] T fbcaa8ec241341a2b56f1e6f7e14f430 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:57.053121 30929 ts_tablet_manager.cc:1398] T fbcaa8ec241341a2b56f1e6f7e14f430 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:57.053244 31068 raft_consensus.cc:348] T c9cc4e72c4f242eab0620c1869bf4c5f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.053366 31068 raft_consensus.cc:374] T c9cc4e72c4f242eab0620c1869bf4c5f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.053401 31068 raft_consensus.cc:729] T c9cc4e72c4f242eab0620c1869bf4c5f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.053356 31072 raft_consensus.cc:348] T 0a28754835d5412ab4ec96600bd8d2a4 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:57.053452 31072 raft_consensus.cc:374] T 0a28754835d5412ab4ec96600bd8d2a4 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.053480 31072 raft_consensus.cc:729] T 0a28754835d5412ab4ec96600bd8d2a4 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.053503 31068 consensus_queue.cc:260] T c9cc4e72c4f242eab0620c1869bf4c5f P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.053581 31072 consensus_queue.cc:260] T 0a28754835d5412ab4ec96600bd8d2a4 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:57.053748 31068 ts_tablet_manager.cc:1398] T c9cc4e72c4f242eab0620c1869bf4c5f P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:57.053910 31072 ts_tablet_manager.cc:1398] T 0a28754835d5412ab4ec96600bd8d2a4 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:57.054064 31072 tablet_bootstrap.cc:492] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:57.054955 31072 tablet_bootstrap.cc:654] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.056134 30884 consensus_queue.cc:1035] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:57.058236 31072 tablet_bootstrap.cc:492] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:57.058320 31072 ts_tablet_manager.cc:1367] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:57.058655 31072 raft_consensus.cc:348] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:57.058751 31072 raft_consensus.cc:374] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.058815 31072 raft_consensus.cc:729] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.058943 31072 consensus_queue.cc:260] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:57.059260 31072 ts_tablet_manager.cc:1398] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:57.060642 30987 consensus_queue.cc:1035] T 5303b025ea014dbeb652cab8f75fb5eb P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:57.062005 22442 catalog_manager.cc:5484] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:57.065033 31028 tablet_bootstrap.cc:492] T fbcaa8ec241341a2b56f1e6f7e14f430 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:57.065140 31028 ts_tablet_manager.cc:1367] T fbcaa8ec241341a2b56f1e6f7e14f430 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.021s user 0.002s sys 0.000s
I20240502 07:58:57.065575 24872 raft_consensus.cc:2454] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:58:57.065691 31028 raft_consensus.cc:348] T fbcaa8ec241341a2b56f1e6f7e14f430 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:57.065798 31028 raft_consensus.cc:374] T fbcaa8ec241341a2b56f1e6f7e14f430 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.065841 31028 raft_consensus.cc:729] T fbcaa8ec241341a2b56f1e6f7e14f430 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.065972 31028 consensus_queue.cc:260] T fbcaa8ec241341a2b56f1e6f7e14f430 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:57.066284 31028 ts_tablet_manager.cc:1398] T fbcaa8ec241341a2b56f1e6f7e14f430 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:57.068511 31071 tablet_bootstrap.cc:492] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
W20240502 07:58:57.068861 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:57.068609 31071 ts_tablet_manager.cc:1367] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.031s user 0.002s sys 0.000s
I20240502 07:58:57.069739 31071 raft_consensus.cc:348] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:57.069833 31071 raft_consensus.cc:374] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.069860 31071 raft_consensus.cc:729] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.069979 31071 consensus_queue.cc:260] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:57.070220 31071 ts_tablet_manager.cc:1398] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:57.073637 31022 raft_consensus.cc:482] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:57.073727 31022 raft_consensus.cc:504] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.074116 31022 leader_election.cc:288] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:57.074416 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cc4e72c4f242eab0620c1869bf4c5f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:57.074478 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cc4e72c4f242eab0620c1869bf4c5f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:57.074548 22716 raft_consensus.cc:2454] T c9cc4e72c4f242eab0620c1869bf4c5f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:57.074605 25133 raft_consensus.cc:2454] T c9cc4e72c4f242eab0620c1869bf4c5f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:57.074811 22912 leader_election.cc:302] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:57.074963 31022 raft_consensus.cc:2781] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:57.075022 31022 raft_consensus.cc:482] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:57.075059 31022 raft_consensus.cc:3037] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.075881 31022 raft_consensus.cc:504] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.076153 31022 leader_election.cc:288] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:57.076411 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cc4e72c4f242eab0620c1869bf4c5f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:57.076454 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9cc4e72c4f242eab0620c1869bf4c5f" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:57.076510 22716 raft_consensus.cc:3037] T c9cc4e72c4f242eab0620c1869bf4c5f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.076542 25133 raft_consensus.cc:3037] T c9cc4e72c4f242eab0620c1869bf4c5f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.077260 22716 raft_consensus.cc:2454] T c9cc4e72c4f242eab0620c1869bf4c5f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:57.077512 22912 leader_election.cc:302] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:57.077685 31022 raft_consensus.cc:2781] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:57.077760 31022 raft_consensus.cc:686] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:57.077890 31022 consensus_queue.cc:237] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.080343 22440 catalog_manager.cc:5484] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:57.101475 25133 raft_consensus.cc:2454] T c9cc4e72c4f242eab0620c1869bf4c5f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:57.114437 30573 raft_consensus.cc:482] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:57.114558 30573 raft_consensus.cc:504] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:57.115032 30573 leader_election.cc:288] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:57.115363 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0a28754835d5412ab4ec96600bd8d2a4" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:57.115363 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0a28754835d5412ab4ec96600bd8d2a4" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:57.115492 24872 raft_consensus.cc:2454] T 0a28754835d5412ab4ec96600bd8d2a4 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:57.115492 22847 raft_consensus.cc:2454] T 0a28754835d5412ab4ec96600bd8d2a4 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:57.115720 25202 leader_election.cc:302] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:57.115871 30573 raft_consensus.cc:2781] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:57.115936 30573 raft_consensus.cc:482] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:57.115984 30573 raft_consensus.cc:3037] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.116822 30573 raft_consensus.cc:504] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:57.117110 30573 leader_election.cc:288] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:58:57.117389 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0a28754835d5412ab4ec96600bd8d2a4" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:57.117410 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0a28754835d5412ab4ec96600bd8d2a4" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:57.117493 24872 raft_consensus.cc:3037] T 0a28754835d5412ab4ec96600bd8d2a4 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.117494 22847 raft_consensus.cc:3037] T 0a28754835d5412ab4ec96600bd8d2a4 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.118201 24872 raft_consensus.cc:2454] T 0a28754835d5412ab4ec96600bd8d2a4 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:57.118274 22847 raft_consensus.cc:2454] T 0a28754835d5412ab4ec96600bd8d2a4 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:57.118450 25199 leader_election.cc:302] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:57.118623 30573 raft_consensus.cc:2781] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:57.118706 30573 raft_consensus.cc:686] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:57.118841 30573 consensus_queue.cc:237] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:57.120954 22440 catalog_manager.cc:5484] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:57.133971 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:58:57.168359 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:57.172247 22586 raft_consensus.cc:1261] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:57.172425 24872 raft_consensus.cc:1261] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:57.172865 30987 consensus_queue.cc:1035] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:57.173115 30987 consensus_queue.cc:1035] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:57.177037 31106 tablet.cc:1722] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:57.177708 22444 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 4241ac2fc3844670b048965ff580e06b (table rebalancer_dynamic_table [id=5bf11aec70844146a3363d737c4444bf]). Expected version 1 got 0
I20240502 07:58:57.178543 31106 tablet.cc:1620] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:57.180370 22440 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet 4241ac2fc3844670b048965ff580e06b (table rebalancer_dynamic_table [id=5bf11aec70844146a3363d737c4444bf]). Expected version 1 got 0
I20240502 07:58:57.180745 22444 catalog_manager.cc:5576] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451) does not have the latest schema for tablet 4241ac2fc3844670b048965ff580e06b (table rebalancer_dynamic_table [id=5bf11aec70844146a3363d737c4444bf]). Expected version 1 got 0
I20240502 07:58:57.180969 31107 tablet.cc:1722] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:57.181268 31107 tablet.cc:1620] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:58:57.181771 22442 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=5bf11aec70844146a3363d737c4444bf] alter complete (version 1)
I20240502 07:58:57.183910 22442 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet 4241ac2fc3844670b048965ff580e06b (table rebalancer_dynamic_table [id=5bf11aec70844146a3363d737c4444bf]). Expected version 1 got 0
I20240502 07:58:57.186934 31113 tablet.cc:1722] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:57.187283 31113 tablet.cc:1620] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
W20240502 07:58:57.226372 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 07:58:57.234531 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 07:58:57.260463 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:58:57.283797 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:57.288715 31026 raft_consensus.cc:482] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:57.288717 31106 tablet.cc:1722] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:57.288837 31026 raft_consensus.cc:504] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:57.289043 31106 tablet.cc:1620] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:58:57.289172 31113 tablet.cc:1722] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:57.289280 31026 leader_election.cc:288] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:57.289458 31113 tablet.cc:1620] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:57.290575 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fbcaa8ec241341a2b56f1e6f7e14f430" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:58:57.290704 24872 raft_consensus.cc:2454] T fbcaa8ec241341a2b56f1e6f7e14f430 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:57.290794 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fbcaa8ec241341a2b56f1e6f7e14f430" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:57.290964 22586 raft_consensus.cc:2454] T fbcaa8ec241341a2b56f1e6f7e14f430 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:57.292162 22784 leader_election.cc:302] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:57.292833 31107 tablet.cc:1722] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:57.293154 31107 tablet.cc:1620] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:58:57.293540 31026 raft_consensus.cc:2781] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:57.293618 31026 raft_consensus.cc:482] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:57.293668 31026 raft_consensus.cc:3037] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.294510 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=5bf11aec70844146a3363d737c4444bf] alter complete (version 2)
I20240502 07:58:57.294646 31026 raft_consensus.cc:504] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:57.294998 31026 leader_election.cc:288] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:57.296444 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fbcaa8ec241341a2b56f1e6f7e14f430" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:58:57.296554 24872 raft_consensus.cc:3037] T fbcaa8ec241341a2b56f1e6f7e14f430 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.296537 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fbcaa8ec241341a2b56f1e6f7e14f430" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:57.296705 22584 raft_consensus.cc:3037] T fbcaa8ec241341a2b56f1e6f7e14f430 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.297523 24872 raft_consensus.cc:2454] T fbcaa8ec241341a2b56f1e6f7e14f430 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:57.297537 22584 raft_consensus.cc:2454] T fbcaa8ec241341a2b56f1e6f7e14f430 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:57.297884 22781 leader_election.cc:302] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:57.298206 31026 raft_consensus.cc:2781] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:57.298358 31026 raft_consensus.cc:686] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:57.298492 31026 consensus_queue.cc:237] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:57.301532 22440 catalog_manager.cc:5484] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:57.346668 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:57.396571 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:57.401881 31107 tablet.cc:1722] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:57.401918 31106 tablet.cc:1722] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:57.402431 31113 tablet.cc:1722] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:57.407398 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=5bf11aec70844146a3363d737c4444bf] alter complete (version 3)
W20240502 07:58:57.408190 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 07:58:57.421540 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 07:58:57.504351 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:58:57.508322 22442 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:57.508446 22442 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:57.510152 22442 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 4241ac2fc3844670b048965ff580e06b
I20240502 07:58:57.510663 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet 4241ac2fc3844670b048965ff580e06b with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:57 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:57.510792 25114 tablet_service.cc:1514] Processing DeleteTablet for tablet 4241ac2fc3844670b048965ff580e06b with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:57 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:58:57.510854 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 4241ac2fc3844670b048965ff580e06b with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:57 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:57.510983 31157 tablet_replica.cc:330] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:57.511119 31157 raft_consensus.cc:2229] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:57.511137 31159 tablet_replica.cc:330] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:57.511188 31158 tablet_replica.cc:330] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:58:57.511255 31157 raft_consensus.cc:2258] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:57.511269 31159 raft_consensus.cc:2229] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:57.511338 31158 raft_consensus.cc:2229] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:57.511376 31159 raft_consensus.cc:2258] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:57.511772 31158 raft_consensus.cc:2258] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:57.511843 31157 ts_tablet_manager.cc:1875] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:57.512027 31159 ts_tablet_manager.cc:1875] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:57.512323 31158 ts_tablet_manager.cc:1875] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:57.514103 31157 ts_tablet_manager.cc:1888] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:57.514225 31157 log.cc:1196] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/4241ac2fc3844670b048965ff580e06b
I20240502 07:58:57.514216 31159 ts_tablet_manager.cc:1888] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:57.514284 31159 log.cc:1196] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/4241ac2fc3844670b048965ff580e06b
I20240502 07:58:57.514514 31157 ts_tablet_manager.cc:1909] T 4241ac2fc3844670b048965ff580e06b P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:57.514631 31159 ts_tablet_manager.cc:1909] T 4241ac2fc3844670b048965ff580e06b P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:57.514618 31158 ts_tablet_manager.cc:1888] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:57.514691 31158 log.cc:1196] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/4241ac2fc3844670b048965ff580e06b
I20240502 07:58:57.514951 31158 ts_tablet_manager.cc:1909] T 4241ac2fc3844670b048965ff580e06b P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:58:57.515205 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet 4241ac2fc3844670b048965ff580e06b (table rebalancer_dynamic_table_new_name [id=5bf11aec70844146a3363d737c4444bf]) successfully deleted
I20240502 07:58:57.515345 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 4241ac2fc3844670b048965ff580e06b (table rebalancer_dynamic_table_new_name [id=5bf11aec70844146a3363d737c4444bf]) successfully deleted
I20240502 07:58:57.515571 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet 4241ac2fc3844670b048965ff580e06b (table rebalancer_dynamic_table_new_name [id=5bf11aec70844146a3363d737c4444bf]) successfully deleted
W20240502 07:58:57.518703 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:58:57.558882 30959 consensus_queue.cc:1035] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:57.561851 30573 consensus_queue.cc:1035] T 0a28754835d5412ab4ec96600bd8d2a4 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:58:57.597679 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:58:57.611629 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:57.615352 31022 consensus_queue.cc:1035] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:57.618121 22566 tablet_service.cc:1467] Processing CreateTablet for tablet bd9c671f604b47438855f7869790c25b (DEFAULT_TABLE table=rebalancer_dynamic_table [id=373f6319d27840e091724f548d08a746]), partition=
I20240502 07:58:57.618257 25235 tablet_service.cc:1467] Processing CreateTablet for tablet bd9c671f604b47438855f7869790c25b (DEFAULT_TABLE table=rebalancer_dynamic_table [id=373f6319d27840e091724f548d08a746]), partition=
I20240502 07:58:57.618376 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd9c671f604b47438855f7869790c25b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.618458 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd9c671f604b47438855f7869790c25b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.620651 31168 tablet_bootstrap.cc:492] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:57.621030 22695 tablet_service.cc:1467] Processing CreateTablet for tablet bd9c671f604b47438855f7869790c25b (DEFAULT_TABLE table=rebalancer_dynamic_table [id=373f6319d27840e091724f548d08a746]), partition=
I20240502 07:58:57.621266 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd9c671f604b47438855f7869790c25b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.621479 30685 consensus_queue.cc:1035] T c9cc4e72c4f242eab0620c1869bf4c5f P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:57.622164 31168 tablet_bootstrap.cc:654] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.623620 31167 tablet_bootstrap.cc:492] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:57.624785 31172 tablet_bootstrap.cc:492] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:57.624908 31167 tablet_bootstrap.cc:654] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.626206 31168 tablet_bootstrap.cc:492] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:57.626268 31172 tablet_bootstrap.cc:654] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.626293 31168 ts_tablet_manager.cc:1367] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:58:57.626770 31168 raft_consensus.cc:348] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:57.626891 31168 raft_consensus.cc:374] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.626925 31168 raft_consensus.cc:729] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.627049 31168 consensus_queue.cc:260] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:57.627174 31172 tablet_bootstrap.cc:492] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:57.627240 31172 ts_tablet_manager.cc:1367] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:57.627314 31168 ts_tablet_manager.cc:1398] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:57.627628 31172 raft_consensus.cc:348] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:57.627736 31172 raft_consensus.cc:374] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.627776 31172 raft_consensus.cc:729] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.627883 31172 consensus_queue.cc:260] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:57.628126 31172 ts_tablet_manager.cc:1398] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:57.628147 31167 tablet_bootstrap.cc:492] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:57.628212 31167 ts_tablet_manager.cc:1367] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.002s
I20240502 07:58:57.628563 31167 raft_consensus.cc:348] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:57.628640 31167 raft_consensus.cc:374] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.628683 31167 raft_consensus.cc:729] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.628787 31167 consensus_queue.cc:260] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:57.629010 31167 ts_tablet_manager.cc:1398] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:57.686417 31027 consensus_queue.cc:1035] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:57.689173 31206 consensus_queue.cc:1035] T fbcaa8ec241341a2b56f1e6f7e14f430 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:58:57.692831 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 07:58:57.767694 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20240502 07:58:57.772009 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 07:58:57.825750 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:58:57.831751 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:58:57.834481 22440 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_40"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:57.840005 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 633d2718fd1c45c6961d3cbd214a2b6d (DEFAULT_TABLE table=rebalancer_extra_table_40 [id=b40b95599ab941b2852b67eecd587ab4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.840092 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 633d2718fd1c45c6961d3cbd214a2b6d (DEFAULT_TABLE table=rebalancer_extra_table_40 [id=b40b95599ab941b2852b67eecd587ab4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.840307 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 633d2718fd1c45c6961d3cbd214a2b6d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.840324 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 633d2718fd1c45c6961d3cbd214a2b6d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.840709 22959 tablet_service.cc:1467] Processing CreateTablet for tablet fb6d3bad102248cda2582cbf7193f860 (DEFAULT_TABLE table=rebalancer_extra_table_40 [id=b40b95599ab941b2852b67eecd587ab4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.840920 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fb6d3bad102248cda2582cbf7193f860. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.843008 22695 tablet_service.cc:1467] Processing CreateTablet for tablet fb6d3bad102248cda2582cbf7193f860 (DEFAULT_TABLE table=rebalancer_extra_table_40 [id=b40b95599ab941b2852b67eecd587ab4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.843117 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 72e339a624ec4a5e9dca7c025906c444 (DEFAULT_TABLE table=rebalancer_extra_table_40 [id=b40b95599ab941b2852b67eecd587ab4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.843247 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fb6d3bad102248cda2582cbf7193f860. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.843202 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 72e339a624ec4a5e9dca7c025906c444 (DEFAULT_TABLE table=rebalancer_extra_table_40 [id=b40b95599ab941b2852b67eecd587ab4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.843341 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 72e339a624ec4a5e9dca7c025906c444. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.843411 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 72e339a624ec4a5e9dca7c025906c444. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.843484 25114 tablet_service.cc:1467] Processing CreateTablet for tablet 633d2718fd1c45c6961d3cbd214a2b6d (DEFAULT_TABLE table=rebalancer_extra_table_40 [id=b40b95599ab941b2852b67eecd587ab4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.843729 25114 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 633d2718fd1c45c6961d3cbd214a2b6d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.845492 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 72e339a624ec4a5e9dca7c025906c444 (DEFAULT_TABLE table=rebalancer_extra_table_40 [id=b40b95599ab941b2852b67eecd587ab4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.845613 31208 tablet_bootstrap.cc:492] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:57.845697 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 72e339a624ec4a5e9dca7c025906c444. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.846113 25113 tablet_service.cc:1467] Processing CreateTablet for tablet fb6d3bad102248cda2582cbf7193f860 (DEFAULT_TABLE table=rebalancer_extra_table_40 [id=b40b95599ab941b2852b67eecd587ab4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:57.846282 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fb6d3bad102248cda2582cbf7193f860. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:57.846520 31208 tablet_bootstrap.cc:654] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.847160 31209 tablet_bootstrap.cc:492] T 633d2718fd1c45c6961d3cbd214a2b6d P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:57.847584 31168 tablet_bootstrap.cc:492] T 72e339a624ec4a5e9dca7c025906c444 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:57.847616 31208 tablet_bootstrap.cc:492] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:57.847679 31208 ts_tablet_manager.cc:1367] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:57.848002 31172 tablet_bootstrap.cc:492] T fb6d3bad102248cda2582cbf7193f860 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:57.848188 31208 raft_consensus.cc:348] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.848279 31208 raft_consensus.cc:374] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.848443 31168 tablet_bootstrap.cc:654] T 72e339a624ec4a5e9dca7c025906c444 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.848879 31172 tablet_bootstrap.cc:654] T fb6d3bad102248cda2582cbf7193f860 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.848306 31208 raft_consensus.cc:729] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.849419 31208 consensus_queue.cc:260] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.849704 31208 ts_tablet_manager.cc:1398] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:57.849732 31168 tablet_bootstrap.cc:492] T 72e339a624ec4a5e9dca7c025906c444 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:57.849799 31168 ts_tablet_manager.cc:1367] T 72e339a624ec4a5e9dca7c025906c444 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.002s
I20240502 07:58:57.849890 31208 tablet_bootstrap.cc:492] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:57.850209 31168 raft_consensus.cc:348] T 72e339a624ec4a5e9dca7c025906c444 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.850314 31168 raft_consensus.cc:374] T 72e339a624ec4a5e9dca7c025906c444 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.850342 31168 raft_consensus.cc:729] T 72e339a624ec4a5e9dca7c025906c444 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.850400 31172 tablet_bootstrap.cc:492] T fb6d3bad102248cda2582cbf7193f860 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:57.850471 31172 ts_tablet_manager.cc:1367] T fb6d3bad102248cda2582cbf7193f860 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:57.850448 31168 consensus_queue.cc:260] T 72e339a624ec4a5e9dca7c025906c444 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.850646 31208 tablet_bootstrap.cc:654] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.850713 31168 ts_tablet_manager.cc:1398] T 72e339a624ec4a5e9dca7c025906c444 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:57.850942 31172 raft_consensus.cc:348] T fb6d3bad102248cda2582cbf7193f860 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:57.851042 31172 raft_consensus.cc:374] T fb6d3bad102248cda2582cbf7193f860 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.851074 31172 raft_consensus.cc:729] T fb6d3bad102248cda2582cbf7193f860 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.851233 31209 tablet_bootstrap.cc:654] T 633d2718fd1c45c6961d3cbd214a2b6d P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.851181 31172 consensus_queue.cc:260] T fb6d3bad102248cda2582cbf7193f860 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:57.851488 31172 ts_tablet_manager.cc:1398] T fb6d3bad102248cda2582cbf7193f860 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:57.852571 31210 tablet_bootstrap.cc:492] T 633d2718fd1c45c6961d3cbd214a2b6d P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:57.853626 31210 tablet_bootstrap.cc:654] T 633d2718fd1c45c6961d3cbd214a2b6d P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.854898 31209 tablet_bootstrap.cc:492] T 633d2718fd1c45c6961d3cbd214a2b6d P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:57.855013 31209 ts_tablet_manager.cc:1367] T 633d2718fd1c45c6961d3cbd214a2b6d P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.000s
I20240502 07:58:57.855523 30573 raft_consensus.cc:482] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:57.855612 30573 raft_consensus.cc:504] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:57.855892 31210 tablet_bootstrap.cc:492] T 633d2718fd1c45c6961d3cbd214a2b6d P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:57.855967 31210 ts_tablet_manager.cc:1367] T 633d2718fd1c45c6961d3cbd214a2b6d P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:57.855986 30573 leader_election.cc:288] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:57.856331 31210 raft_consensus.cc:348] T 633d2718fd1c45c6961d3cbd214a2b6d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.856426 31210 raft_consensus.cc:374] T 633d2718fd1c45c6961d3cbd214a2b6d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.856452 31210 raft_consensus.cc:729] T 633d2718fd1c45c6961d3cbd214a2b6d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.856555 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd9c671f604b47438855f7869790c25b" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:57.856576 31210 consensus_queue.cc:260] T 633d2718fd1c45c6961d3cbd214a2b6d P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.856595 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd9c671f604b47438855f7869790c25b" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:57.856674 22586 raft_consensus.cc:2454] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:57.856698 22716 raft_consensus.cc:2454] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:58:57.856879 31210 ts_tablet_manager.cc:1398] T 633d2718fd1c45c6961d3cbd214a2b6d P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:57.856930 25202 leader_election.cc:302] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7eced220cf1642f3873fc385b6191e69, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:57.856999 31210 tablet_bootstrap.cc:492] T 72e339a624ec4a5e9dca7c025906c444 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:57.857079 30573 raft_consensus.cc:2781] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:57.857122 30573 raft_consensus.cc:482] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:57.857156 30573 raft_consensus.cc:3037] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.857925 31210 tablet_bootstrap.cc:654] T 72e339a624ec4a5e9dca7c025906c444 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.858156 30573 raft_consensus.cc:504] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:57.858428 31208 tablet_bootstrap.cc:492] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:57.858493 31208 ts_tablet_manager.cc:1367] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.009s user 0.001s sys 0.000s
I20240502 07:58:57.858480 30573 leader_election.cc:288] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:57.858693 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd9c671f604b47438855f7869790c25b" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:57.858778 22586 raft_consensus.cc:3037] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.858975 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd9c671f604b47438855f7869790c25b" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:57.859064 22716 raft_consensus.cc:3037] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.859241 31208 raft_consensus.cc:348] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:57.859362 31208 raft_consensus.cc:374] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.859416 31208 raft_consensus.cc:729] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.859563 31208 consensus_queue.cc:260] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:57.859647 22586 raft_consensus.cc:2454] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:57.859906 31208 ts_tablet_manager.cc:1398] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:57.859925 22716 raft_consensus.cc:2454] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:58:57.859968 25202 leader_election.cc:302] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7eced220cf1642f3873fc385b6191e69, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:57.861140 30573 raft_consensus.cc:2781] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:57.861240 30573 raft_consensus.cc:686] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:58:57.861356 30573 consensus_queue.cc:237] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
W20240502 07:58:57.861879 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:58:57.863257 22442 catalog_manager.cc:5484] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:57.863797 31209 raft_consensus.cc:348] T 633d2718fd1c45c6961d3cbd214a2b6d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.863904 31209 raft_consensus.cc:374] T 633d2718fd1c45c6961d3cbd214a2b6d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.863960 31209 raft_consensus.cc:729] T 633d2718fd1c45c6961d3cbd214a2b6d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.864061 31209 consensus_queue.cc:260] T 633d2718fd1c45c6961d3cbd214a2b6d P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.864351 31209 ts_tablet_manager.cc:1398] T 633d2718fd1c45c6961d3cbd214a2b6d P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.009s user 0.001s sys 0.000s
I20240502 07:58:57.864701 31209 tablet_bootstrap.cc:492] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:57.865610 31209 tablet_bootstrap.cc:654] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.866597 31209 tablet_bootstrap.cc:492] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:57.866675 31209 ts_tablet_manager.cc:1367] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:57.867010 31209 raft_consensus.cc:348] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.867097 31209 raft_consensus.cc:374] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.867132 31209 raft_consensus.cc:729] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.867269 31209 consensus_queue.cc:260] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.867528 31209 ts_tablet_manager.cc:1398] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:57.868027 31210 tablet_bootstrap.cc:492] T 72e339a624ec4a5e9dca7c025906c444 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:57.868106 31210 ts_tablet_manager.cc:1367] T 72e339a624ec4a5e9dca7c025906c444 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.011s user 0.000s sys 0.001s
I20240502 07:58:57.868412 31210 raft_consensus.cc:348] T 72e339a624ec4a5e9dca7c025906c444 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.868492 31210 raft_consensus.cc:374] T 72e339a624ec4a5e9dca7c025906c444 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.868521 31210 raft_consensus.cc:729] T 72e339a624ec4a5e9dca7c025906c444 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.868606 31210 consensus_queue.cc:260] T 72e339a624ec4a5e9dca7c025906c444 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.868877 31210 ts_tablet_manager.cc:1398] T 72e339a624ec4a5e9dca7c025906c444 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:57.869024 31210 tablet_bootstrap.cc:492] T fb6d3bad102248cda2582cbf7193f860 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:57.869957 31210 tablet_bootstrap.cc:654] T fb6d3bad102248cda2582cbf7193f860 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:57.870945 31210 tablet_bootstrap.cc:492] T fb6d3bad102248cda2582cbf7193f860 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:57.871026 31210 ts_tablet_manager.cc:1367] T fb6d3bad102248cda2582cbf7193f860 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:57.871318 31210 raft_consensus.cc:348] T fb6d3bad102248cda2582cbf7193f860 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:57.871387 31210 raft_consensus.cc:374] T fb6d3bad102248cda2582cbf7193f860 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:57.871404 31210 raft_consensus.cc:729] T fb6d3bad102248cda2582cbf7193f860 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:57.871488 31210 consensus_queue.cc:260] T fb6d3bad102248cda2582cbf7193f860 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:57.871730 31210 ts_tablet_manager.cc:1398] T fb6d3bad102248cda2582cbf7193f860 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
W20240502 07:58:57.878568 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:58:57.966575 30685 raft_consensus.cc:482] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:57.966689 30685 raft_consensus.cc:504] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:57.967123 30685 leader_election.cc:288] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:57.967422 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb6d3bad102248cda2582cbf7193f860" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:57.967453 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb6d3bad102248cda2582cbf7193f860" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:57.967557 22716 raft_consensus.cc:2454] T fb6d3bad102248cda2582cbf7193f860 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:57.967551 25135 raft_consensus.cc:2454] T fb6d3bad102248cda2582cbf7193f860 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:57.967782 22912 leader_election.cc:302] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:57.967907 30685 raft_consensus.cc:2781] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:57.967955 30685 raft_consensus.cc:482] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:57.967993 30685 raft_consensus.cc:3037] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.968799 30685 raft_consensus.cc:504] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:57.969067 30685 leader_election.cc:288] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:57.969342 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb6d3bad102248cda2582cbf7193f860" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:57.969425 25135 raft_consensus.cc:3037] T fb6d3bad102248cda2582cbf7193f860 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.969354 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb6d3bad102248cda2582cbf7193f860" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:57.969503 22716 raft_consensus.cc:3037] T fb6d3bad102248cda2582cbf7193f860 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.970108 25135 raft_consensus.cc:2454] T fb6d3bad102248cda2582cbf7193f860 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:57.970212 22716 raft_consensus.cc:2454] T fb6d3bad102248cda2582cbf7193f860 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:57.970443 22912 leader_election.cc:302] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:57.970569 30685 raft_consensus.cc:2781] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:57.970649 30685 raft_consensus.cc:686] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:57.970791 30685 consensus_queue.cc:237] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:57.972944 22442 catalog_manager.cc:5484] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:57.981348 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20240502 07:58:57.981604 31206 raft_consensus.cc:482] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:57.981699 31206 raft_consensus.cc:504] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.982043 31206 leader_election.cc:288] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:57.982308 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "72e339a624ec4a5e9dca7c025906c444" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:57.982311 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "72e339a624ec4a5e9dca7c025906c444" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:57.982404 25135 raft_consensus.cc:2454] T 72e339a624ec4a5e9dca7c025906c444 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:57.982427 22586 raft_consensus.cc:2454] T 72e339a624ec4a5e9dca7c025906c444 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:57.982646 22784 leader_election.cc:302] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:57.982812 31206 raft_consensus.cc:2781] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:57.982870 31206 raft_consensus.cc:482] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:57.982909 31206 raft_consensus.cc:3037] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.984015 31206 raft_consensus.cc:504] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.984265 31206 leader_election.cc:288] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:57.984560 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "72e339a624ec4a5e9dca7c025906c444" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:57.984611 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "72e339a624ec4a5e9dca7c025906c444" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:57.984685 22586 raft_consensus.cc:3037] T 72e339a624ec4a5e9dca7c025906c444 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.984753 25135 raft_consensus.cc:3037] T 72e339a624ec4a5e9dca7c025906c444 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:57.985589 22586 raft_consensus.cc:2454] T 72e339a624ec4a5e9dca7c025906c444 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:57.985597 25135 raft_consensus.cc:2454] T 72e339a624ec4a5e9dca7c025906c444 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:57.985813 22784 leader_election.cc:302] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:57.985946 31206 raft_consensus.cc:2781] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:57.986023 31206 raft_consensus.cc:686] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:57.986142 31206 consensus_queue.cc:237] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:57.987974 22442 catalog_manager.cc:5484] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:58.014459 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:58.017879 22716 raft_consensus.cc:1261] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:58.017943 22586 raft_consensus.cc:1261] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:58.018227 30573 consensus_queue.cc:1035] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:58.018357 30959 consensus_queue.cc:1035] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:58.020735 22440 catalog_manager.cc:5576] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451) does not have the latest schema for tablet bd9c671f604b47438855f7869790c25b (table rebalancer_dynamic_table [id=373f6319d27840e091724f548d08a746]). Expected version 1 got 0
I20240502 07:58:58.021277 31256 tablet.cc:1722] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:58.022303 31256 tablet.cc:1620] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:58:58.022341 22440 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet bd9c671f604b47438855f7869790c25b (table rebalancer_dynamic_table [id=373f6319d27840e091724f548d08a746]). Expected version 1 got 0
I20240502 07:58:58.022697 22444 catalog_manager.cc:5576] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451) does not have the latest schema for tablet bd9c671f604b47438855f7869790c25b (table rebalancer_dynamic_table [id=373f6319d27840e091724f548d08a746]). Expected version 1 got 0
I20240502 07:58:58.023370 31255 tablet.cc:1722] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:58.023617 31255 tablet.cc:1620] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:58.023912 22444 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet bd9c671f604b47438855f7869790c25b (table rebalancer_dynamic_table [id=373f6319d27840e091724f548d08a746]). Expected version 1 got 0
I20240502 07:58:58.026101 22442 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet bd9c671f604b47438855f7869790c25b (table rebalancer_dynamic_table [id=373f6319d27840e091724f548d08a746]). Expected version 1 got 0
I20240502 07:58:58.027065 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=373f6319d27840e091724f548d08a746] alter complete (version 1)
I20240502 07:58:58.027518 31258 tablet.cc:1722] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:58.027941 31258 tablet.cc:1620] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:58:58.031394 30685 raft_consensus.cc:482] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:58.031492 30685 raft_consensus.cc:504] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:58.031893 30685 leader_election.cc:288] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:58.032218 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "633d2718fd1c45c6961d3cbd214a2b6d" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:58.032301 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "633d2718fd1c45c6961d3cbd214a2b6d" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:58.032337 22847 raft_consensus.cc:2454] T 633d2718fd1c45c6961d3cbd214a2b6d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:58.032423 25135 raft_consensus.cc:2454] T 633d2718fd1c45c6961d3cbd214a2b6d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:58.032584 22915 leader_election.cc:302] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:58.032766 30685 raft_consensus.cc:2781] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:58.032829 30685 raft_consensus.cc:482] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:58.032879 30685 raft_consensus.cc:3037] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:58.033836 30685 raft_consensus.cc:504] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:58.034081 30685 leader_election.cc:288] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:58.034363 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "633d2718fd1c45c6961d3cbd214a2b6d" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:58.034468 22847 raft_consensus.cc:3037] T 633d2718fd1c45c6961d3cbd214a2b6d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:58.034461 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "633d2718fd1c45c6961d3cbd214a2b6d" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:58.034572 25135 raft_consensus.cc:3037] T 633d2718fd1c45c6961d3cbd214a2b6d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:58.035393 22847 raft_consensus.cc:2454] T 633d2718fd1c45c6961d3cbd214a2b6d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:58.035516 25135 raft_consensus.cc:2454] T 633d2718fd1c45c6961d3cbd214a2b6d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:58.035669 22915 leader_election.cc:302] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:58.035811 30685 raft_consensus.cc:2781] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:58.035894 30685 raft_consensus.cc:686] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:58.036005 30685 consensus_queue.cc:237] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:58.037729 22440 catalog_manager.cc:5484] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:58.130749 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:58.136978 31255 tablet.cc:1722] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:58.137310 31255 tablet.cc:1620] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:58.138049 31256 tablet.cc:1722] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:58.138305 31256 tablet.cc:1620] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:58:58.141098 31258 tablet.cc:1722] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:58.141388 31258 tablet.cc:1620] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:58:58.143280 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=373f6319d27840e091724f548d08a746] alter complete (version 2)
W20240502 07:58:58.171765 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 07:58:58.220729 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:58.246316 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:58.250984 31255 tablet.cc:1722] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:58.251323 31258 tablet.cc:1722] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:58.253154 31256 tablet.cc:1722] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:58.256044 22442 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=373f6319d27840e091724f548d08a746] alter complete (version 3)
W20240502 07:58:58.291083 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:58:58.357609 22442 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:58.357748 22442 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:58.359608 22442 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet bd9c671f604b47438855f7869790c25b
I20240502 07:58:58.360131 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet bd9c671f604b47438855f7869790c25b with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:58 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:58.360473 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet bd9c671f604b47438855f7869790c25b with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:58 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:58.360785 31291 tablet_replica.cc:330] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:58.360747 25235 tablet_service.cc:1514] Processing DeleteTablet for tablet bd9c671f604b47438855f7869790c25b with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:58 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:58.360920 31291 raft_consensus.cc:2229] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:58.361064 31291 raft_consensus.cc:2258] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:58.361140 31292 tablet_replica.cc:330] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:58.361160 31293 tablet_replica.cc:330] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:58.361280 31293 raft_consensus.cc:2229] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:58.361325 31292 raft_consensus.cc:2229] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:58.361482 31292 raft_consensus.cc:2258] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:58.361543 31293 raft_consensus.cc:2258] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:58.361753 31291 ts_tablet_manager.cc:1875] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:58.362263 31293 ts_tablet_manager.cc:1875] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:58.362311 31292 ts_tablet_manager.cc:1875] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:58.364127 31291 ts_tablet_manager.cc:1888] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:58.364281 31291 log.cc:1196] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/bd9c671f604b47438855f7869790c25b
I20240502 07:58:58.364676 31293 ts_tablet_manager.cc:1888] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:58.364713 31291 ts_tablet_manager.cc:1909] T bd9c671f604b47438855f7869790c25b P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:58.364784 31293 log.cc:1196] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/bd9c671f604b47438855f7869790c25b
I20240502 07:58:58.365073 31293 ts_tablet_manager.cc:1909] T bd9c671f604b47438855f7869790c25b P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:58:58.365653 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet bd9c671f604b47438855f7869790c25b (table rebalancer_dynamic_table_new_name [id=373f6319d27840e091724f548d08a746]) successfully deleted
I20240502 07:58:58.367681 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet bd9c671f604b47438855f7869790c25b (table rebalancer_dynamic_table_new_name [id=373f6319d27840e091724f548d08a746]) successfully deleted
I20240502 07:58:58.370997 31292 ts_tablet_manager.cc:1888] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:58.371054 31292 log.cc:1196] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/bd9c671f604b47438855f7869790c25b
I20240502 07:58:58.371282 31292 ts_tablet_manager.cc:1909] T bd9c671f604b47438855f7869790c25b P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:58.371801 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet bd9c671f604b47438855f7869790c25b (table rebalancer_dynamic_table_new_name [id=373f6319d27840e091724f548d08a746]) successfully deleted
I20240502 07:58:58.376919 31290 consensus_queue.cc:1035] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:58.389382 31027 consensus_queue.cc:1035] T 72e339a624ec4a5e9dca7c025906c444 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:58.400465 30685 consensus_queue.cc:1035] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:58.403730 31022 consensus_queue.cc:1035] T fb6d3bad102248cda2582cbf7193f860 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:58:58.433705 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 5487e66cb4d8416094a1321cb989f018: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:58.462260 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:58.466912 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 6871455dd8b340eaac68b5000c91be71 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=7274e662f4f44bdba77b8700172bc97d]), partition=
I20240502 07:58:58.467078 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 6871455dd8b340eaac68b5000c91be71 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=7274e662f4f44bdba77b8700172bc97d]), partition=
I20240502 07:58:58.467156 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6871455dd8b340eaac68b5000c91be71. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:58.467265 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6871455dd8b340eaac68b5000c91be71. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:58.468978 31334 tablet_bootstrap.cc:492] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:58.468992 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 6871455dd8b340eaac68b5000c91be71 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=7274e662f4f44bdba77b8700172bc97d]), partition=
I20240502 07:58:58.469226 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6871455dd8b340eaac68b5000c91be71. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:58.470654 31334 tablet_bootstrap.cc:654] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:58.471153 31335 tablet_bootstrap.cc:492] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:58.471423 31336 tablet_bootstrap.cc:492] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:58.471954 31335 tablet_bootstrap.cc:654] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:58.472338 31334 tablet_bootstrap.cc:492] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:58.472400 31336 tablet_bootstrap.cc:654] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:58.472539 31334 ts_tablet_manager.cc:1367] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:58.473048 31334 raft_consensus.cc:348] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:58.473167 31334 raft_consensus.cc:374] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:58.473208 31334 raft_consensus.cc:729] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:58.473248 31335 tablet_bootstrap.cc:492] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:58.473317 31335 ts_tablet_manager.cc:1367] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:58.473321 31334 consensus_queue.cc:260] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:58.473743 31335 raft_consensus.cc:348] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:58.473851 31335 raft_consensus.cc:374] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:58.473879 31335 raft_consensus.cc:729] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:58.473979 31335 consensus_queue.cc:260] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:58.474253 31335 ts_tablet_manager.cc:1398] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:58.474764 31336 tablet_bootstrap.cc:492] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:58.474902 31336 ts_tablet_manager.cc:1367] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:58:58.475368 31336 raft_consensus.cc:348] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:58.475462 31334 ts_tablet_manager.cc:1398] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:58.475483 31336 raft_consensus.cc:374] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:58.475512 31336 raft_consensus.cc:729] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:58.475638 31336 consensus_queue.cc:260] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:58.475945 31336 ts_tablet_manager.cc:1398] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:58.479311 31038 raft_consensus.cc:482] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:58.479385 31038 raft_consensus.cc:504] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:58.479694 31038 leader_election.cc:288] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:58.479912 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6871455dd8b340eaac68b5000c91be71" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:58.479935 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6871455dd8b340eaac68b5000c91be71" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:58.480018 25265 raft_consensus.cc:2454] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:58.480018 22586 raft_consensus.cc:2454] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:58:58.480245 22652 leader_election.cc:302] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:58.480410 31038 raft_consensus.cc:2781] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:58.480455 31038 raft_consensus.cc:482] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:58.480495 31038 raft_consensus.cc:3037] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:58.481199 31038 raft_consensus.cc:504] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:58.481428 31038 leader_election.cc:288] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:58.481683 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6871455dd8b340eaac68b5000c91be71" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:58.481740 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6871455dd8b340eaac68b5000c91be71" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:58.481765 25265 raft_consensus.cc:3037] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:58.481843 22586 raft_consensus.cc:3037] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:58.482422 25265 raft_consensus.cc:2454] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:58.482501 22586 raft_consensus.cc:2454] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:58:58.482662 22652 leader_election.cc:302] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:58.482815 31038 raft_consensus.cc:2781] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:58.482888 31038 raft_consensus.cc:686] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:58:58.483032 31038 consensus_queue.cc:237] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:58.484727 22442 catalog_manager.cc:5484] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:58.504243 31022 consensus_queue.cc:1035] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:58.507087 31342 consensus_queue.cc:1035] T 633d2718fd1c45c6961d3cbd214a2b6d P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:58:58.516119 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 07:58:58.526926 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 07:58:58.543443 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:58:58.579416 22442 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_42"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:58.585105 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 4e51b6ab240b46a990571d86548a0970 (DEFAULT_TABLE table=rebalancer_extra_table_42 [id=f1c246ad39b74450ac1c6814d4db34b5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:58.585429 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4e51b6ab240b46a990571d86548a0970. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:58.586057 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 3bacf9b518374c13b0f21dad2b7bceba (DEFAULT_TABLE table=rebalancer_extra_table_42 [id=f1c246ad39b74450ac1c6814d4db34b5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:58.586138 25246 tablet_service.cc:1467] Processing CreateTablet for tablet ec66d621f66f4cb3b275ff6f829cfb34 (DEFAULT_TABLE table=rebalancer_extra_table_42 [id=f1c246ad39b74450ac1c6814d4db34b5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:58.586292 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3bacf9b518374c13b0f21dad2b7bceba. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:58.586373 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ec66d621f66f4cb3b275ff6f829cfb34. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:58.586616 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 4e51b6ab240b46a990571d86548a0970 (DEFAULT_TABLE table=rebalancer_extra_table_42 [id=f1c246ad39b74450ac1c6814d4db34b5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:58.586846 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4e51b6ab240b46a990571d86548a0970. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:58.587710 25113 tablet_service.cc:1467] Processing CreateTablet for tablet ec66d621f66f4cb3b275ff6f829cfb34 (DEFAULT_TABLE table=rebalancer_extra_table_42 [id=f1c246ad39b74450ac1c6814d4db34b5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:58.587936 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ec66d621f66f4cb3b275ff6f829cfb34. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:58.588425 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 3bacf9b518374c13b0f21dad2b7bceba (DEFAULT_TABLE table=rebalancer_extra_table_42 [id=f1c246ad39b74450ac1c6814d4db34b5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:58.588694 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3bacf9b518374c13b0f21dad2b7bceba. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:58.588932 31335 tablet_bootstrap.cc:492] T ec66d621f66f4cb3b275ff6f829cfb34 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:58.589145 22442 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:58.589742 31335 tablet_bootstrap.cc:654] T ec66d621f66f4cb3b275ff6f829cfb34 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:58.590121 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 4e51b6ab240b46a990571d86548a0970 (DEFAULT_TABLE table=rebalancer_extra_table_42 [id=f1c246ad39b74450ac1c6814d4db34b5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:58.590304 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4e51b6ab240b46a990571d86548a0970. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:58.590721 31336 tablet_bootstrap.cc:492] T 3bacf9b518374c13b0f21dad2b7bceba P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:58.591652 31336 tablet_bootstrap.cc:654] T 3bacf9b518374c13b0f21dad2b7bceba P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:58.592324 31334 tablet_bootstrap.cc:492] T 3bacf9b518374c13b0f21dad2b7bceba P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:58.592311 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 3bacf9b518374c13b0f21dad2b7bceba (DEFAULT_TABLE table=rebalancer_extra_table_42 [id=f1c246ad39b74450ac1c6814d4db34b5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:58.592531 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3bacf9b518374c13b0f21dad2b7bceba. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:58.592903 31335 tablet_bootstrap.cc:492] T ec66d621f66f4cb3b275ff6f829cfb34 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:58.592969 31335 ts_tablet_manager.cc:1367] T ec66d621f66f4cb3b275ff6f829cfb34 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:58:58.593228 31334 tablet_bootstrap.cc:654] T 3bacf9b518374c13b0f21dad2b7bceba P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:58.593381 31335 raft_consensus.cc:348] T ec66d621f66f4cb3b275ff6f829cfb34 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:58.593465 31335 raft_consensus.cc:374] T ec66d621f66f4cb3b275ff6f829cfb34 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:58.593498 31335 raft_consensus.cc:729] T ec66d621f66f4cb3b275ff6f829cfb34 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:58.593605 31335 consensus_queue.cc:260] T ec66d621f66f4cb3b275ff6f829cfb34 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:58.593864 31335 ts_tablet_manager.cc:1398] T ec66d621f66f4cb3b275ff6f829cfb34 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:58.593993 31335 tablet_bootstrap.cc:492] T 4e51b6ab240b46a990571d86548a0970 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:58.594137 31334 tablet_bootstrap.cc:492] T 3bacf9b518374c13b0f21dad2b7bceba P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:58.594219 31334 ts_tablet_manager.cc:1367] T 3bacf9b518374c13b0f21dad2b7bceba P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:58.594249 22957 tablet_service.cc:1467] Processing CreateTablet for tablet ec66d621f66f4cb3b275ff6f829cfb34 (DEFAULT_TABLE table=rebalancer_extra_table_42 [id=f1c246ad39b74450ac1c6814d4db34b5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:58.594478 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ec66d621f66f4cb3b275ff6f829cfb34. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:58.594666 31334 raft_consensus.cc:348] T 3bacf9b518374c13b0f21dad2b7bceba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:58.594770 31334 raft_consensus.cc:374] T 3bacf9b518374c13b0f21dad2b7bceba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:58.594802 31334 raft_consensus.cc:729] T 3bacf9b518374c13b0f21dad2b7bceba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:58.594907 31334 consensus_queue.cc:260] T 3bacf9b518374c13b0f21dad2b7bceba P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:58.594945 31335 tablet_bootstrap.cc:654] T 4e51b6ab240b46a990571d86548a0970 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:58.595142 31334 ts_tablet_manager.cc:1398] T 3bacf9b518374c13b0f21dad2b7bceba P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:58.597741 31336 tablet_bootstrap.cc:492] T 3bacf9b518374c13b0f21dad2b7bceba P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:58.597826 31336 ts_tablet_manager.cc:1367] T 3bacf9b518374c13b0f21dad2b7bceba P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
I20240502 07:58:58.598232 31336 raft_consensus.cc:348] T 3bacf9b518374c13b0f21dad2b7bceba P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:58.598331 31336 raft_consensus.cc:374] T 3bacf9b518374c13b0f21dad2b7bceba P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:58.598361 31336 raft_consensus.cc:729] T 3bacf9b518374c13b0f21dad2b7bceba P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:58.598455 31336 consensus_queue.cc:260] T 3bacf9b518374c13b0f21dad2b7bceba P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:58.598512 31344 tablet_bootstrap.cc:492] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:58.598696 31336 ts_tablet_manager.cc:1398] T 3bacf9b518374c13b0f21dad2b7bceba P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:58.599248 31344 tablet_bootstrap.cc:654] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:58.600301 31344 tablet_bootstrap.cc:492] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:58.600378 31344 ts_tablet_manager.cc:1367] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:58.600783 31344 raft_consensus.cc:348] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:58.600838 22584 raft_consensus.cc:1261] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:58.600894 31344 raft_consensus.cc:374] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:58.600926 31344 raft_consensus.cc:729] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:58.601016 31344 consensus_queue.cc:260] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:58.601162 31038 consensus_queue.cc:1035] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:58.601248 31344 ts_tablet_manager.cc:1398] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:58.601367 31344 tablet_bootstrap.cc:492] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:58.601843 31346 tablet_bootstrap.cc:492] T ec66d621f66f4cb3b275ff6f829cfb34 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:58.602074 31344 tablet_bootstrap.cc:654] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:58.602365 25265 raft_consensus.cc:1261] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:58.602550 31335 tablet_bootstrap.cc:492] T 4e51b6ab240b46a990571d86548a0970 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:58.602623 31335 ts_tablet_manager.cc:1367] T 4e51b6ab240b46a990571d86548a0970 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.009s user 0.000s sys 0.002s
I20240502 07:58:58.602892 31038 consensus_queue.cc:1035] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:58.602995 31335 raft_consensus.cc:348] T 4e51b6ab240b46a990571d86548a0970 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:58.603077 31335 raft_consensus.cc:374] T 4e51b6ab240b46a990571d86548a0970 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:58.603103 31335 raft_consensus.cc:729] T 4e51b6ab240b46a990571d86548a0970 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:58.603127 31345 tablet_bootstrap.cc:492] T 4e51b6ab240b46a990571d86548a0970 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:58.603199 31335 consensus_queue.cc:260] T 4e51b6ab240b46a990571d86548a0970 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:58.603430 31335 ts_tablet_manager.cc:1398] T 4e51b6ab240b46a990571d86548a0970 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:58.603971 31345 tablet_bootstrap.cc:654] T 4e51b6ab240b46a990571d86548a0970 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:58.604642 22442 catalog_manager.cc:5576] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451) does not have the latest schema for tablet 6871455dd8b340eaac68b5000c91be71 (table rebalancer_dynamic_table [id=7274e662f4f44bdba77b8700172bc97d]). Expected version 1 got 0
I20240502 07:58:58.605659 31258 tablet.cc:1722] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:58.605906 31258 tablet.cc:1620] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:58:58.606463 22436 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet 6871455dd8b340eaac68b5000c91be71 (table rebalancer_dynamic_table [id=7274e662f4f44bdba77b8700172bc97d]). Expected version 1 got 0
I20240502 07:58:58.606725 31346 tablet_bootstrap.cc:654] T ec66d621f66f4cb3b275ff6f829cfb34 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:58.606853 31256 tablet.cc:1722] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:58.607110 31256 tablet.cc:1620] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:58:58.608286 31344 tablet_bootstrap.cc:492] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:58.608354 31346 tablet_bootstrap.cc:492] T ec66d621f66f4cb3b275ff6f829cfb34 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:58.608361 31344 ts_tablet_manager.cc:1367] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 07:58:58.608422 31346 ts_tablet_manager.cc:1367] T ec66d621f66f4cb3b275ff6f829cfb34 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 07:58:58.608767 31344 raft_consensus.cc:348] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:58.608875 31344 raft_consensus.cc:374] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:58.608901 31344 raft_consensus.cc:729] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:58.609011 31344 consensus_queue.cc:260] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:58.609251 31344 ts_tablet_manager.cc:1398] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:58.609380 31255 tablet.cc:1722] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:58.609611 31255 tablet.cc:1620] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:58.610503 31346 raft_consensus.cc:348] T ec66d621f66f4cb3b275ff6f829cfb34 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:58.610653 31346 raft_consensus.cc:374] T ec66d621f66f4cb3b275ff6f829cfb34 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:58.610724 31346 raft_consensus.cc:729] T ec66d621f66f4cb3b275ff6f829cfb34 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:58.610936 31346 consensus_queue.cc:260] T ec66d621f66f4cb3b275ff6f829cfb34 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:58.612603 31345 tablet_bootstrap.cc:492] T 4e51b6ab240b46a990571d86548a0970 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:58.612812 31345 ts_tablet_manager.cc:1367] T 4e51b6ab240b46a990571d86548a0970 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.000s
I20240502 07:58:58.613224 22444 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet 6871455dd8b340eaac68b5000c91be71 (table rebalancer_dynamic_table [id=7274e662f4f44bdba77b8700172bc97d]). Expected version 1 got 0
I20240502 07:58:58.613243 31345 raft_consensus.cc:348] T 4e51b6ab240b46a990571d86548a0970 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:58.613335 31345 raft_consensus.cc:374] T 4e51b6ab240b46a990571d86548a0970 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:58.613363 31345 raft_consensus.cc:729] T 4e51b6ab240b46a990571d86548a0970 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:58.613463 31345 consensus_queue.cc:260] T 4e51b6ab240b46a990571d86548a0970 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:58.613719 31345 ts_tablet_manager.cc:1398] T 4e51b6ab240b46a990571d86548a0970 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:58.613919 31345 tablet_bootstrap.cc:492] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:58.614931 31345 tablet_bootstrap.cc:654] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:58.616029 31345 tablet_bootstrap.cc:492] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:58.616102 31345 ts_tablet_manager.cc:1367] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:58.616485 31345 raft_consensus.cc:348] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:58.616678 31345 raft_consensus.cc:374] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:58.616569 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=7274e662f4f44bdba77b8700172bc97d] alter complete (version 1)
I20240502 07:58:58.616727 31345 raft_consensus.cc:729] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:58.616883 31345 consensus_queue.cc:260] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:58.617245 31345 ts_tablet_manager.cc:1398] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:58.617604 31346 ts_tablet_manager.cc:1398] T ec66d621f66f4cb3b275ff6f829cfb34 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.009s user 0.001s sys 0.000s
I20240502 07:58:58.644985 31358 raft_consensus.cc:482] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:58.645112 31358 raft_consensus.cc:504] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:58.645606 31358 leader_election.cc:288] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:58.645888 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3bacf9b518374c13b0f21dad2b7bceba" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:58.646031 22716 raft_consensus.cc:2454] T 3bacf9b518374c13b0f21dad2b7bceba P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:58.646159 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3bacf9b518374c13b0f21dad2b7bceba" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:58.646310 22584 raft_consensus.cc:2454] T 3bacf9b518374c13b0f21dad2b7bceba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:58.646310 24805 leader_election.cc:302] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:58.646853 31358 raft_consensus.cc:2781] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:58.646929 31358 raft_consensus.cc:482] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:58.646977 31358 raft_consensus.cc:3037] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:58.648111 31358 raft_consensus.cc:504] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:58.648418 31358 leader_election.cc:288] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:58.648710 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3bacf9b518374c13b0f21dad2b7bceba" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
W20240502 07:58:58.648764 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:58:58.648833 22716 raft_consensus.cc:3037] T 3bacf9b518374c13b0f21dad2b7bceba P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:58.649740 22716 raft_consensus.cc:2454] T 3bacf9b518374c13b0f21dad2b7bceba P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:58.650130 24805 leader_election.cc:302] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:58:58.650326 31358 raft_consensus.cc:2781] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:58.650429 31358 raft_consensus.cc:686] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:58:58.650558 31358 consensus_queue.cc:237] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:58:58.653286 22438 catalog_manager.cc:5484] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:58.653717 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3bacf9b518374c13b0f21dad2b7bceba" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:58.653828 22584 raft_consensus.cc:3037] T 3bacf9b518374c13b0f21dad2b7bceba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:58.654701 22584 raft_consensus.cc:2454] T 3bacf9b518374c13b0f21dad2b7bceba P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
W20240502 07:58:58.676059 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:58:58.720372 22438 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:58.725652 31258 tablet.cc:1722] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:58.725960 31258 tablet.cc:1620] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:58:58.726858 31255 tablet.cc:1722] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:58.727103 31255 tablet.cc:1620] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:58:58.727674 31256 tablet.cc:1722] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:58.727931 31256 tablet.cc:1620] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:58:58.733537 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=7274e662f4f44bdba77b8700172bc97d] alter complete (version 2)
I20240502 07:58:58.771723 31342 raft_consensus.cc:482] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:58.771834 31342 raft_consensus.cc:504] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:58.772274 31342 leader_election.cc:288] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:58.772536 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ec66d621f66f4cb3b275ff6f829cfb34" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:58.772575 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ec66d621f66f4cb3b275ff6f829cfb34" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:58.772701 25135 raft_consensus.cc:2454] T ec66d621f66f4cb3b275ff6f829cfb34 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:58.772716 25266 raft_consensus.cc:2454] T ec66d621f66f4cb3b275ff6f829cfb34 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:58.772969 22914 leader_election.cc:302] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:58.773170 31342 raft_consensus.cc:2781] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:58.773224 31342 raft_consensus.cc:482] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:58.773252 31342 raft_consensus.cc:3037] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:58.774302 31342 raft_consensus.cc:504] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:58.774642 31342 leader_election.cc:288] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:58.774956 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ec66d621f66f4cb3b275ff6f829cfb34" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:58.775045 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ec66d621f66f4cb3b275ff6f829cfb34" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:58.775079 25135 raft_consensus.cc:3037] T ec66d621f66f4cb3b275ff6f829cfb34 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:58.775153 25266 raft_consensus.cc:3037] T ec66d621f66f4cb3b275ff6f829cfb34 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:58.775967 25266 raft_consensus.cc:2454] T ec66d621f66f4cb3b275ff6f829cfb34 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:58.775972 25135 raft_consensus.cc:2454] T ec66d621f66f4cb3b275ff6f829cfb34 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:58.776216 22915 leader_election.cc:302] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 4654231dfe994c1a9d2e37d9c3d75699; no voters:
I20240502 07:58:58.776437 31342 raft_consensus.cc:2781] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:58.776512 31342 raft_consensus.cc:686] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:58.776648 31342 consensus_queue.cc:237] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:58.778743 22444 catalog_manager.cc:5484] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:58.787251 31358 raft_consensus.cc:482] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:58.787341 31358 raft_consensus.cc:504] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:58.787705 31358 leader_election.cc:288] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:58.787966 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4e51b6ab240b46a990571d86548a0970" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:58.788025 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4e51b6ab240b46a990571d86548a0970" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:58.788066 25266 raft_consensus.cc:2454] T 4e51b6ab240b46a990571d86548a0970 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:58.788130 22978 raft_consensus.cc:2454] T 4e51b6ab240b46a990571d86548a0970 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:58.788311 24807 leader_election.cc:302] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:58.788455 31358 raft_consensus.cc:2781] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:58.788508 31358 raft_consensus.cc:482] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:58.788530 31358 raft_consensus.cc:3037] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:58.789278 31358 raft_consensus.cc:504] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:58.789520 31358 leader_election.cc:288] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:58:58.789824 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4e51b6ab240b46a990571d86548a0970" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:58.789844 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4e51b6ab240b46a990571d86548a0970" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:58.789935 22978 raft_consensus.cc:3037] T 4e51b6ab240b46a990571d86548a0970 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:58.789935 25266 raft_consensus.cc:3037] T 4e51b6ab240b46a990571d86548a0970 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:58.790757 25266 raft_consensus.cc:2454] T 4e51b6ab240b46a990571d86548a0970 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:58.790773 22978 raft_consensus.cc:2454] T 4e51b6ab240b46a990571d86548a0970 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:58.791067 24806 leader_election.cc:302] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:58:58.791237 31358 raft_consensus.cc:2781] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:58.791318 31358 raft_consensus.cc:686] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:58:58.791443 31358 consensus_queue.cc:237] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:58:58.793398 22444 catalog_manager.cc:5484] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:58.836302 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
W20240502 07:58:58.837893 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:58.841400 31258 tablet.cc:1722] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:58.842204 31255 tablet.cc:1722] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:58.843492 31256 tablet.cc:1722] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:58.847360 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=7274e662f4f44bdba77b8700172bc97d] alter complete (version 3)
W20240502 07:58:58.904745 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 07:58:58.942596 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:58:58.948925 22444 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:58.949041 22444 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:58.950744 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 6871455dd8b340eaac68b5000c91be71
I20240502 07:58:58.951300 25235 tablet_service.cc:1514] Processing DeleteTablet for tablet 6871455dd8b340eaac68b5000c91be71 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:58 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:58:58.951300 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 6871455dd8b340eaac68b5000c91be71 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:58 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:58:58.951591 31423 tablet_replica.cc:330] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:58:58.951560 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet 6871455dd8b340eaac68b5000c91be71 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:58 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:58.951757 31423 raft_consensus.cc:2229] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:58.951829 31425 tablet_replica.cc:330] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:58.951925 31423 raft_consensus.cc:2258] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:58.951949 31425 raft_consensus.cc:2229] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:58.952138 31425 raft_consensus.cc:2258] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:58.952168 31424 tablet_replica.cc:330] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:58:58.952289 31424 raft_consensus.cc:2229] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:58.952445 31424 raft_consensus.cc:2258] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:58.952589 31423 ts_tablet_manager.cc:1875] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:58.952963 31425 ts_tablet_manager.cc:1875] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:58.953047 31424 ts_tablet_manager.cc:1875] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:58.954723 31423 ts_tablet_manager.cc:1888] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:58.954836 31423 log.cc:1196] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/6871455dd8b340eaac68b5000c91be71
I20240502 07:58:58.955085 31425 ts_tablet_manager.cc:1888] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:58.955147 31423 ts_tablet_manager.cc:1909] T 6871455dd8b340eaac68b5000c91be71 P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:58:58.955199 31425 log.cc:1196] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/6871455dd8b340eaac68b5000c91be71
I20240502 07:58:58.955168 31424 ts_tablet_manager.cc:1888] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:58.955252 31424 log.cc:1196] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/6871455dd8b340eaac68b5000c91be71
I20240502 07:58:58.955600 31424 ts_tablet_manager.cc:1909] T 6871455dd8b340eaac68b5000c91be71 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:58:58.955559 31425 ts_tablet_manager.cc:1909] T 6871455dd8b340eaac68b5000c91be71 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:58.955818 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet 6871455dd8b340eaac68b5000c91be71 (table rebalancer_dynamic_table_new_name [id=7274e662f4f44bdba77b8700172bc97d]) successfully deleted
I20240502 07:58:58.956177 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 6871455dd8b340eaac68b5000c91be71 (table rebalancer_dynamic_table_new_name [id=7274e662f4f44bdba77b8700172bc97d]) successfully deleted
I20240502 07:58:58.956317 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 6871455dd8b340eaac68b5000c91be71 (table rebalancer_dynamic_table_new_name [id=7274e662f4f44bdba77b8700172bc97d]) successfully deleted
W20240502 07:58:59.043884 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:58:59.052318 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:59.057698 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 4f9e506492994fbf96c61bb803d35291 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=d8ac6160e0e6475b92e23f953939e7b0]), partition=
I20240502 07:58:59.057705 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 4f9e506492994fbf96c61bb803d35291 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=d8ac6160e0e6475b92e23f953939e7b0]), partition=
I20240502 07:58:59.057951 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f9e506492994fbf96c61bb803d35291. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:59.057951 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f9e506492994fbf96c61bb803d35291. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:59.059983 31344 tablet_bootstrap.cc:492] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:59.060149 31426 tablet_bootstrap.cc:492] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:59.060956 31344 tablet_bootstrap.cc:654] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:59.061257 31426 tablet_bootstrap.cc:654] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:59.061334 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 4f9e506492994fbf96c61bb803d35291 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=d8ac6160e0e6475b92e23f953939e7b0]), partition=
I20240502 07:58:59.061568 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f9e506492994fbf96c61bb803d35291. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:59.062213 31426 tablet_bootstrap.cc:492] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:59.062294 31426 ts_tablet_manager.cc:1367] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.002s
I20240502 07:58:59.062696 31426 raft_consensus.cc:348] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.062803 31426 raft_consensus.cc:374] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:59.062844 31426 raft_consensus.cc:729] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:59.062955 31426 consensus_queue.cc:260] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.063217 31426 ts_tablet_manager.cc:1398] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:58:59.063378 31336 tablet_bootstrap.cc:492] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:59.064237 31336 tablet_bootstrap.cc:654] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:59.065178 31344 tablet_bootstrap.cc:492] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:59.065268 31344 ts_tablet_manager.cc:1367] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:58:59.065802 31344 raft_consensus.cc:348] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.065861 31336 tablet_bootstrap.cc:492] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:59.065904 31344 raft_consensus.cc:374] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:59.065935 31344 raft_consensus.cc:729] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:59.065937 31336 ts_tablet_manager.cc:1367] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:58:59.066033 31344 consensus_queue.cc:260] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.066282 31344 ts_tablet_manager.cc:1398] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:59.066270 31336 raft_consensus.cc:348] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.066340 31336 raft_consensus.cc:374] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:59.066359 31336 raft_consensus.cc:729] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:59.066449 31336 consensus_queue.cc:260] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.066686 31336 ts_tablet_manager.cc:1398] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
W20240502 07:58:59.075824 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:58:59.080046 31358 consensus_queue.cc:1035] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:59.082516 31358 consensus_queue.cc:1035] T 3bacf9b518374c13b0f21dad2b7bceba P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:59.155144 31022 consensus_queue.cc:1035] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:59.158346 31342 consensus_queue.cc:1035] T ec66d621f66f4cb3b275ff6f829cfb34 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:59.162228 31358 raft_consensus.cc:482] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:59.162321 31358 raft_consensus.cc:504] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.162673 31358 leader_election.cc:288] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:59.162952 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f9e506492994fbf96c61bb803d35291" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:59.162993 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f9e506492994fbf96c61bb803d35291" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:59.163072 22847 raft_consensus.cc:2454] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:59.163115 22717 raft_consensus.cc:2454] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:59.163369 24808 leader_election.cc:302] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:59.163568 31358 raft_consensus.cc:2781] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:59.163604 31358 raft_consensus.cc:482] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:59.163640 31358 raft_consensus.cc:3037] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:59.164474 31358 raft_consensus.cc:504] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.164883 31358 leader_election.cc:288] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:59.165249 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f9e506492994fbf96c61bb803d35291" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:59.165364 22847 raft_consensus.cc:3037] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:59.165521 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f9e506492994fbf96c61bb803d35291" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:59.165627 22717 raft_consensus.cc:3037] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:59.166363 22847 raft_consensus.cc:2454] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:59.166487 22717 raft_consensus.cc:2454] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:59.166605 24808 leader_election.cc:302] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:59.166797 31163 raft_consensus.cc:2781] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:59.166887 31163 raft_consensus.cc:686] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:58:59.167003 31163 consensus_queue.cc:237] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.168788 22436 catalog_manager.cc:5484] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:59.198772 31163 consensus_queue.cc:1035] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:59.201493 31163 consensus_queue.cc:1035] T 4e51b6ab240b46a990571d86548a0970 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:59.284021 22438 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:59.288050 22717 raft_consensus.cc:1261] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:59.288374 22847 raft_consensus.cc:1261] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:59.288574 31358 consensus_queue.cc:1035] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:59.288743 31163 consensus_queue.cc:1035] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:59.291209 31475 tablet.cc:1722] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:59.291518 31475 tablet.cc:1620] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:59.291643 22438 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet 4f9e506492994fbf96c61bb803d35291 (table rebalancer_dynamic_table [id=d8ac6160e0e6475b92e23f953939e7b0]). Expected version 1 got 0
I20240502 07:58:59.292258 31258 tablet.cc:1722] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:59.292505 31258 tablet.cc:1620] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:58:59.293182 22436 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet 4f9e506492994fbf96c61bb803d35291 (table rebalancer_dynamic_table [id=d8ac6160e0e6475b92e23f953939e7b0]). Expected version 1 got 0
I20240502 07:58:59.294451 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=d8ac6160e0e6475b92e23f953939e7b0] alter complete (version 1)
I20240502 07:58:59.295029 31476 tablet.cc:1722] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:59.295316 31476 tablet.cc:1620] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:58:59.296468 22444 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet 4f9e506492994fbf96c61bb803d35291 (table rebalancer_dynamic_table [id=d8ac6160e0e6475b92e23f953939e7b0]). Expected version 1 got 0
I20240502 07:58:59.299261 22438 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_44"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:58:59.303611 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 4c08ecf9d52a4a419b2f52251968d9ab (DEFAULT_TABLE table=rebalancer_extra_table_44 [id=ad3fa189302d4f5e828aec73c6bf019b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:59.303836 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 4c08ecf9d52a4a419b2f52251968d9ab (DEFAULT_TABLE table=rebalancer_extra_table_44 [id=ad3fa189302d4f5e828aec73c6bf019b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:59.303905 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4c08ecf9d52a4a419b2f52251968d9ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:59.304080 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4c08ecf9d52a4a419b2f52251968d9ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:59.304436 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 50f615cdc41b4334b74e130584c89983 (DEFAULT_TABLE table=rebalancer_extra_table_44 [id=ad3fa189302d4f5e828aec73c6bf019b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:59.304657 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 50f615cdc41b4334b74e130584c89983. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:59.305891 31477 tablet_bootstrap.cc:492] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:59.306231 31478 tablet_bootstrap.cc:492] T 4c08ecf9d52a4a419b2f52251968d9ab P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:59.306399 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 4c08ecf9d52a4a419b2f52251968d9ab (DEFAULT_TABLE table=rebalancer_extra_table_44 [id=ad3fa189302d4f5e828aec73c6bf019b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:59.306643 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4c08ecf9d52a4a419b2f52251968d9ab. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:59.307166 31478 tablet_bootstrap.cc:654] T 4c08ecf9d52a4a419b2f52251968d9ab P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:59.307611 31477 tablet_bootstrap.cc:654] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:59.308127 31478 tablet_bootstrap.cc:492] T 4c08ecf9d52a4a419b2f52251968d9ab P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:59.308180 31478 ts_tablet_manager.cc:1367] T 4c08ecf9d52a4a419b2f52251968d9ab P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:59.308465 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 073fa19317844cc4aedaf386261268f7 (DEFAULT_TABLE table=rebalancer_extra_table_44 [id=ad3fa189302d4f5e828aec73c6bf019b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:59.308524 31477 tablet_bootstrap.cc:492] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:59.308596 31477 ts_tablet_manager.cc:1367] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:59.308570 31478 raft_consensus.cc:348] T 4c08ecf9d52a4a419b2f52251968d9ab P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:59.308677 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 073fa19317844cc4aedaf386261268f7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:59.308713 31478 raft_consensus.cc:374] T 4c08ecf9d52a4a419b2f52251968d9ab P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:59.308745 31478 raft_consensus.cc:729] T 4c08ecf9d52a4a419b2f52251968d9ab P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:59.308859 31478 consensus_queue.cc:260] T 4c08ecf9d52a4a419b2f52251968d9ab P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:59.308990 31477 raft_consensus.cc:348] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:59.309069 31477 raft_consensus.cc:374] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:59.309095 31477 raft_consensus.cc:729] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:59.309484 31477 consensus_queue.cc:260] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:59.309861 31478 ts_tablet_manager.cc:1398] T 4c08ecf9d52a4a419b2f52251968d9ab P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:59.310379 31336 tablet_bootstrap.cc:492] T 073fa19317844cc4aedaf386261268f7 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:58:59.310607 31344 tablet_bootstrap.cc:492] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:58:59.311048 31477 ts_tablet_manager.cc:1398] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:59.311288 31336 tablet_bootstrap.cc:654] T 073fa19317844cc4aedaf386261268f7 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:59.311455 31481 tablet_bootstrap.cc:492] T 4c08ecf9d52a4a419b2f52251968d9ab P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:58:59.312472 31336 tablet_bootstrap.cc:492] T 073fa19317844cc4aedaf386261268f7 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:58:59.312536 31336 ts_tablet_manager.cc:1367] T 073fa19317844cc4aedaf386261268f7 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:58:59.312950 31336 raft_consensus.cc:348] T 073fa19317844cc4aedaf386261268f7 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.313056 31336 raft_consensus.cc:374] T 073fa19317844cc4aedaf386261268f7 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:59.313087 31336 raft_consensus.cc:729] T 073fa19317844cc4aedaf386261268f7 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:58:59.313186 31336 consensus_queue.cc:260] T 073fa19317844cc4aedaf386261268f7 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.313427 31336 ts_tablet_manager.cc:1398] T 073fa19317844cc4aedaf386261268f7 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:59.314353 31344 tablet_bootstrap.cc:654] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:59.314919 31481 tablet_bootstrap.cc:654] T 4c08ecf9d52a4a419b2f52251968d9ab P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:59.315620 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 073fa19317844cc4aedaf386261268f7 (DEFAULT_TABLE table=rebalancer_extra_table_44 [id=ad3fa189302d4f5e828aec73c6bf019b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:59.315634 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 073fa19317844cc4aedaf386261268f7 (DEFAULT_TABLE table=rebalancer_extra_table_44 [id=ad3fa189302d4f5e828aec73c6bf019b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:59.315836 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 073fa19317844cc4aedaf386261268f7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:59.315865 31481 tablet_bootstrap.cc:492] T 4c08ecf9d52a4a419b2f52251968d9ab P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:58:59.315882 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 073fa19317844cc4aedaf386261268f7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:59.315922 31481 ts_tablet_manager.cc:1367] T 4c08ecf9d52a4a419b2f52251968d9ab P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:59.315960 31344 tablet_bootstrap.cc:492] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:58:59.316020 31344 ts_tablet_manager.cc:1367] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:58:59.316296 31481 raft_consensus.cc:348] T 4c08ecf9d52a4a419b2f52251968d9ab P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:59.316375 31481 raft_consensus.cc:374] T 4c08ecf9d52a4a419b2f52251968d9ab P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:59.316397 31481 raft_consensus.cc:729] T 4c08ecf9d52a4a419b2f52251968d9ab P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:58:59.316390 31344 raft_consensus.cc:348] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:59.316470 31344 raft_consensus.cc:374] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:59.316493 31344 raft_consensus.cc:729] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:58:59.316485 31481 consensus_queue.cc:260] T 4c08ecf9d52a4a419b2f52251968d9ab P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:59.316596 31344 consensus_queue.cc:260] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:59.316732 31481 ts_tablet_manager.cc:1398] T 4c08ecf9d52a4a419b2f52251968d9ab P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:59.316841 31344 ts_tablet_manager.cc:1398] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:59.317598 31477 tablet_bootstrap.cc:492] T 073fa19317844cc4aedaf386261268f7 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:59.317968 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 50f615cdc41b4334b74e130584c89983 (DEFAULT_TABLE table=rebalancer_extra_table_44 [id=ad3fa189302d4f5e828aec73c6bf019b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:59.318147 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 50f615cdc41b4334b74e130584c89983. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:59.318423 31477 tablet_bootstrap.cc:654] T 073fa19317844cc4aedaf386261268f7 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:59.319113 31426 tablet_bootstrap.cc:492] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:59.315634 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 50f615cdc41b4334b74e130584c89983 (DEFAULT_TABLE table=rebalancer_extra_table_44 [id=ad3fa189302d4f5e828aec73c6bf019b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:58:59.319947 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 50f615cdc41b4334b74e130584c89983. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:59.320006 31426 tablet_bootstrap.cc:654] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:59.321504 31426 tablet_bootstrap.cc:492] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:59.321590 31426 ts_tablet_manager.cc:1367] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240502 07:58:59.321938 31426 raft_consensus.cc:348] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.322026 31426 raft_consensus.cc:374] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:59.322044 31426 raft_consensus.cc:729] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:59.322116 31426 consensus_queue.cc:260] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.322328 31426 ts_tablet_manager.cc:1398] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:58:59.322475 31426 tablet_bootstrap.cc:492] T 50f615cdc41b4334b74e130584c89983 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:59.323240 31426 tablet_bootstrap.cc:654] T 50f615cdc41b4334b74e130584c89983 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:59.327086 31477 tablet_bootstrap.cc:492] T 073fa19317844cc4aedaf386261268f7 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:59.327172 31477 ts_tablet_manager.cc:1367] T 073fa19317844cc4aedaf386261268f7 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.010s user 0.001s sys 0.000s
I20240502 07:58:59.327589 31477 raft_consensus.cc:348] T 073fa19317844cc4aedaf386261268f7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.327699 31477 raft_consensus.cc:374] T 073fa19317844cc4aedaf386261268f7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:59.327730 31477 raft_consensus.cc:729] T 073fa19317844cc4aedaf386261268f7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:59.327826 31477 consensus_queue.cc:260] T 073fa19317844cc4aedaf386261268f7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.328085 31477 ts_tablet_manager.cc:1398] T 073fa19317844cc4aedaf386261268f7 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:59.328223 31477 tablet_bootstrap.cc:492] T 50f615cdc41b4334b74e130584c89983 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:58:59.329005 31477 tablet_bootstrap.cc:654] T 50f615cdc41b4334b74e130584c89983 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:59.330622 31477 tablet_bootstrap.cc:492] T 50f615cdc41b4334b74e130584c89983 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:58:59.330686 31477 ts_tablet_manager.cc:1367] T 50f615cdc41b4334b74e130584c89983 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:59.330986 30708 raft_consensus.cc:482] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:59.331063 30708 raft_consensus.cc:504] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:59.331095 31477 raft_consensus.cc:348] T 50f615cdc41b4334b74e130584c89983 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:59.331211 31477 raft_consensus.cc:374] T 50f615cdc41b4334b74e130584c89983 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:59.331243 31477 raft_consensus.cc:729] T 50f615cdc41b4334b74e130584c89983 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:58:59.331368 31477 consensus_queue.cc:260] T 50f615cdc41b4334b74e130584c89983 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:59.331449 30708 leader_election.cc:288] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:59.331712 31477 ts_tablet_manager.cc:1398] T 50f615cdc41b4334b74e130584c89983 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:59.331781 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4c08ecf9d52a4a419b2f52251968d9ab" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:58:59.331897 31426 tablet_bootstrap.cc:492] T 50f615cdc41b4334b74e130584c89983 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:59.331889 25133 raft_consensus.cc:2454] T 4c08ecf9d52a4a419b2f52251968d9ab P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:59.331956 31426 ts_tablet_manager.cc:1367] T 50f615cdc41b4334b74e130584c89983 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.010s user 0.000s sys 0.001s
I20240502 07:58:59.332161 22522 leader_election.cc:302] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:59.332312 31494 raft_consensus.cc:2781] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:59.332358 31494 raft_consensus.cc:482] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:59.332309 31426 raft_consensus.cc:348] T 50f615cdc41b4334b74e130584c89983 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:59.332406 31426 raft_consensus.cc:374] T 50f615cdc41b4334b74e130584c89983 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:59.332401 31494 raft_consensus.cc:3037] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:59.332451 31426 raft_consensus.cc:729] T 50f615cdc41b4334b74e130584c89983 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:59.332566 31426 consensus_queue.cc:260] T 50f615cdc41b4334b74e130584c89983 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:59.332814 31426 ts_tablet_manager.cc:1398] T 50f615cdc41b4334b74e130584c89983 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:59.333356 31494 raft_consensus.cc:504] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:59.333847 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4c08ecf9d52a4a419b2f52251968d9ab" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:59.333971 25266 raft_consensus.cc:2454] T 4c08ecf9d52a4a419b2f52251968d9ab P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:58:59.334005 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4c08ecf9d52a4a419b2f52251968d9ab" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:58:59.334111 25133 raft_consensus.cc:3037] T 4c08ecf9d52a4a419b2f52251968d9ab P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:59.334596 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4c08ecf9d52a4a419b2f52251968d9ab" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:59.334697 25266 raft_consensus.cc:3037] T 4c08ecf9d52a4a419b2f52251968d9ab P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:59.335130 25133 raft_consensus.cc:2454] T 4c08ecf9d52a4a419b2f52251968d9ab P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:59.335410 22522 leader_election.cc:302] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:59.335577 30708 raft_consensus.cc:2781] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:59.335670 25266 raft_consensus.cc:2454] T 4c08ecf9d52a4a419b2f52251968d9ab P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:58:59.335733 30708 raft_consensus.cc:686] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:58:59.335855 30708 consensus_queue.cc:237] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:58:59.336279 31494 leader_election.cc:288] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:58:59.338598 22438 catalog_manager.cc:5484] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
W20240502 07:58:59.365109 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:59.376212 31163 raft_consensus.cc:482] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:59.376305 31163 raft_consensus.cc:504] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:59.376638 31163 leader_election.cc:288] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:59.376957 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "50f615cdc41b4334b74e130584c89983" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:59.377070 22847 raft_consensus.cc:2454] T 50f615cdc41b4334b74e130584c89983 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:59.377295 24808 leader_election.cc:302] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:59.377460 31163 raft_consensus.cc:2781] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:59.377512 31163 raft_consensus.cc:482] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:59.377489 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "50f615cdc41b4334b74e130584c89983" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:59.377555 31163 raft_consensus.cc:3037] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:59.377607 22584 raft_consensus.cc:2454] T 50f615cdc41b4334b74e130584c89983 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:58:59.378403 31163 raft_consensus.cc:504] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:59.378712 31163 leader_election.cc:288] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:58:59.379284 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "50f615cdc41b4334b74e130584c89983" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:59.379290 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "50f615cdc41b4334b74e130584c89983" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:59.379384 22584 raft_consensus.cc:3037] T 50f615cdc41b4334b74e130584c89983 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:59.379386 22847 raft_consensus.cc:3037] T 50f615cdc41b4334b74e130584c89983 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:59.380131 22847 raft_consensus.cc:2454] T 50f615cdc41b4334b74e130584c89983 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:59.380251 22584 raft_consensus.cc:2454] T 50f615cdc41b4334b74e130584c89983 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:58:59.380399 24808 leader_election.cc:302] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:59.380549 31163 raft_consensus.cc:2781] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:59.380612 31163 raft_consensus.cc:686] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:58:59.380791 31163 consensus_queue.cc:237] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:58:59.382861 22438 catalog_manager.cc:5484] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } }
I20240502 07:58:59.399044 22438 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:58:59.403570 31475 tablet.cc:1722] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:59.403865 31475 tablet.cc:1620] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:58:59.404083 31476 tablet.cc:1722] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:59.404393 31476 tablet.cc:1620] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:58:59.406056 31258 tablet.cc:1722] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:58:59.406316 31258 tablet.cc:1620] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:58:59.408852 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=d8ac6160e0e6475b92e23f953939e7b0] alter complete (version 2)
I20240502 07:58:59.510260 31027 raft_consensus.cc:482] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:59.510357 31027 raft_consensus.cc:504] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.510671 31027 leader_election.cc:288] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:59.510954 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "073fa19317844cc4aedaf386261268f7" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:58:59.510960 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "073fa19317844cc4aedaf386261268f7" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:58:59.511070 22584 raft_consensus.cc:2454] T 073fa19317844cc4aedaf386261268f7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:59.511078 22717 raft_consensus.cc:2454] T 073fa19317844cc4aedaf386261268f7 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:59.511314 22784 leader_election.cc:302] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:59.511464 31027 raft_consensus.cc:2781] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:59.511515 31027 raft_consensus.cc:482] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:59.511547 31027 raft_consensus.cc:3037] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:59.512148 22435 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:58:59.512542 31027 raft_consensus.cc:504] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.512873 31027 leader_election.cc:288] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:58:59.513125 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "073fa19317844cc4aedaf386261268f7" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:58:59.513223 22584 raft_consensus.cc:3037] T 073fa19317844cc4aedaf386261268f7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:59.513273 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "073fa19317844cc4aedaf386261268f7" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:58:59.513376 22717 raft_consensus.cc:3037] T 073fa19317844cc4aedaf386261268f7 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:59.514425 22717 raft_consensus.cc:2454] T 073fa19317844cc4aedaf386261268f7 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:59.514438 22584 raft_consensus.cc:2454] T 073fa19317844cc4aedaf386261268f7 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:58:59.514670 22784 leader_election.cc:302] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:58:59.515033 31027 raft_consensus.cc:2781] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:59.515110 31027 raft_consensus.cc:686] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:58:59.515228 31027 consensus_queue.cc:237] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:58:59.517638 31475 tablet.cc:1722] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:59.518031 22435 catalog_manager.cc:5484] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:59.518388 31476 tablet.cc:1722] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:59.518388 31258 tablet.cc:1722] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:58:59.522300 22438 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=d8ac6160e0e6475b92e23f953939e7b0] alter complete (version 3)
W20240502 07:58:59.570670 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:58:59.623324 22436 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:59.623514 22436 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:58:59.625545 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 4f9e506492994fbf96c61bb803d35291
I20240502 07:58:59.626173 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet 4f9e506492994fbf96c61bb803d35291 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:59 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:58:59.626201 24850 tablet_service.cc:1514] Processing DeleteTablet for tablet 4f9e506492994fbf96c61bb803d35291 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:59 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:58:59.626574 31557 tablet_replica.cc:330] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:58:59.626570 22828 tablet_service.cc:1514] Processing DeleteTablet for tablet 4f9e506492994fbf96c61bb803d35291 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:58:59 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:58:59.626783 31557 raft_consensus.cc:2229] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:59.626857 31558 tablet_replica.cc:330] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:58:59.626922 31557 raft_consensus.cc:2258] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:59.627022 31559 tablet_replica.cc:330] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:58:59.627027 31558 raft_consensus.cc:2229] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:58:59.627246 31558 raft_consensus.cc:2258] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:59.627839 31558 ts_tablet_manager.cc:1875] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:59.627640 31559 raft_consensus.cc:2229] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:58:59.628069 31559 raft_consensus.cc:2258] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:58:59.628556 31557 ts_tablet_manager.cc:1875] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:58:59.628682 31559 ts_tablet_manager.cc:1875] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
W20240502 07:58:59.630345 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:58:59.630473 31558 ts_tablet_manager.cc:1888] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:59.630618 31558 log.cc:1196] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/4f9e506492994fbf96c61bb803d35291
I20240502 07:58:59.631018 31558 ts_tablet_manager.cc:1909] T 4f9e506492994fbf96c61bb803d35291 P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:58:59.631664 31559 ts_tablet_manager.cc:1888] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:59.631752 31559 log.cc:1196] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/4f9e506492994fbf96c61bb803d35291
I20240502 07:58:59.632004 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet 4f9e506492994fbf96c61bb803d35291 (table rebalancer_dynamic_table_new_name [id=d8ac6160e0e6475b92e23f953939e7b0]) successfully deleted
I20240502 07:58:59.632483 31559 ts_tablet_manager.cc:1909] T 4f9e506492994fbf96c61bb803d35291 P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:58:59.633169 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 4f9e506492994fbf96c61bb803d35291 (table rebalancer_dynamic_table_new_name [id=d8ac6160e0e6475b92e23f953939e7b0]) successfully deleted
I20240502 07:58:59.633653 31557 ts_tablet_manager.cc:1888] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:58:59.633715 31557 log.cc:1196] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/4f9e506492994fbf96c61bb803d35291
I20240502 07:58:59.633975 31557 ts_tablet_manager.cc:1909] T 4f9e506492994fbf96c61bb803d35291 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:58:59.634531 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 4f9e506492994fbf96c61bb803d35291 (table rebalancer_dynamic_table_new_name [id=d8ac6160e0e6475b92e23f953939e7b0]) successfully deleted
W20240502 07:58:59.678377 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 07:58:59.713182 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:58:59.727253 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:58:59.732108 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 1f586c18b98b432ebbaadfa2a277a5cc (DEFAULT_TABLE table=rebalancer_dynamic_table [id=ab63bb39038d4b8bb4622d79cfcb9da3]), partition=
I20240502 07:58:59.732131 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 1f586c18b98b432ebbaadfa2a277a5cc (DEFAULT_TABLE table=rebalancer_dynamic_table [id=ab63bb39038d4b8bb4622d79cfcb9da3]), partition=
I20240502 07:58:59.732260 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 1f586c18b98b432ebbaadfa2a277a5cc (DEFAULT_TABLE table=rebalancer_dynamic_table [id=ab63bb39038d4b8bb4622d79cfcb9da3]), partition=
I20240502 07:58:59.732344 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1f586c18b98b432ebbaadfa2a277a5cc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:59.732371 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1f586c18b98b432ebbaadfa2a277a5cc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:59.732493 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1f586c18b98b432ebbaadfa2a277a5cc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:58:59.734329 31426 tablet_bootstrap.cc:492] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:58:59.734469 31478 tablet_bootstrap.cc:492] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:58:59.735186 31426 tablet_bootstrap.cc:654] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:59.735198 31478 tablet_bootstrap.cc:654] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:59.736135 31560 tablet_bootstrap.cc:492] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:58:59.736285 31478 tablet_bootstrap.cc:492] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:58:59.736363 31478 ts_tablet_manager.cc:1367] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:58:59.736771 31478 raft_consensus.cc:348] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:59.736866 31478 raft_consensus.cc:374] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:59.736892 31478 raft_consensus.cc:729] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:58:59.736987 31478 consensus_queue.cc:260] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:59.737092 31560 tablet_bootstrap.cc:654] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:58:59.737212 31478 ts_tablet_manager.cc:1398] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:59.737675 31426 tablet_bootstrap.cc:492] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:58:59.737749 31426 ts_tablet_manager.cc:1367] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:58:59.738126 31426 raft_consensus.cc:348] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:59.738212 31426 raft_consensus.cc:374] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:59.738245 31426 raft_consensus.cc:729] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:58:59.738343 31426 consensus_queue.cc:260] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:59.738559 31426 ts_tablet_manager.cc:1398] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:59.741988 31560 tablet_bootstrap.cc:492] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:58:59.742055 31560 ts_tablet_manager.cc:1367] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:58:59.742363 31560 raft_consensus.cc:348] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:59.742460 31560 raft_consensus.cc:374] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:58:59.742489 31560 raft_consensus.cc:729] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:58:59.742592 31560 consensus_queue.cc:260] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:59.742977 31560 ts_tablet_manager.cc:1398] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:58:59.751572 31494 consensus_queue.cc:1035] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:59.755720 30708 consensus_queue.cc:1035] T 4c08ecf9d52a4a419b2f52251968d9ab P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:59.776288 31027 raft_consensus.cc:482] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:59.776386 31027 raft_consensus.cc:504] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:59.776796 31027 leader_election.cc:288] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:59.776911 30685 raft_consensus.cc:482] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:58:59.776993 30685 raft_consensus.cc:504] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:59.777074 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1f586c18b98b432ebbaadfa2a277a5cc" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:58:59.777153 22978 raft_consensus.cc:2454] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:59.777314 30685 leader_election.cc:288] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:59.777488 22782 leader_election.cc:302] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:58:59.777493 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1f586c18b98b432ebbaadfa2a277a5cc" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:59.777530 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1f586c18b98b432ebbaadfa2a277a5cc" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:58:59.777621 22847 raft_consensus.cc:2454] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:59.777616 25265 raft_consensus.cc:2454] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:58:59.777639 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1f586c18b98b432ebbaadfa2a277a5cc" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:58:59.777700 25266 raft_consensus.cc:2454] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:58:59.777702 31027 raft_consensus.cc:2781] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:59.777740 31027 raft_consensus.cc:482] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:59.777779 31027 raft_consensus.cc:3037] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:59.777920 22914 leader_election.cc:302] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:59.778060 31342 raft_consensus.cc:2781] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:58:59.778127 31342 raft_consensus.cc:482] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:58:59.778157 31342 raft_consensus.cc:3037] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:59.779053 31342 raft_consensus.cc:504] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:59.779217 31027 raft_consensus.cc:504] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:59.779357 31342 leader_election.cc:288] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:59.779718 31027 leader_election.cc:288] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:58:59.779754 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1f586c18b98b432ebbaadfa2a277a5cc" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:58:59.779791 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1f586c18b98b432ebbaadfa2a277a5cc" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:58:59.779839 25266 raft_consensus.cc:3037] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:58:59.779918 22978 raft_consensus.cc:2379] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 7db4178eed174225bfb24f5f9445ce88 in current term 1: Already voted for candidate 02a92e1c084b4308b9e855a4cc2c295f in this term.
W20240502 07:58:59.780454 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:58:59.780732 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1f586c18b98b432ebbaadfa2a277a5cc" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:58:59.780752 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1f586c18b98b432ebbaadfa2a277a5cc" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
W20240502 07:58:59.780797 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:58:59.780859 22847 raft_consensus.cc:2379] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 02a92e1c084b4308b9e855a4cc2c295f in current term 1: Already voted for candidate 7db4178eed174225bfb24f5f9445ce88 in this term.
I20240502 07:58:59.780891 25266 raft_consensus.cc:2454] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:58:59.781067 22914 leader_election.cc:302] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:58:59.781106 22783 leader_election.cc:302] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88; no voters: 02a92e1c084b4308b9e855a4cc2c295f, 7eced220cf1642f3873fc385b6191e69
I20240502 07:58:59.781231 31342 raft_consensus.cc:2781] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:58:59.781261 31027 raft_consensus.cc:2726] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20240502 07:58:59.781307 31342 raft_consensus.cc:686] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:58:59.781442 31342 consensus_queue.cc:237] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:58:59.783272 22435 catalog_manager.cc:5484] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
I20240502 07:58:59.881794 31525 consensus_queue.cc:1035] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:59.885567 31525 consensus_queue.cc:1035] T 50f615cdc41b4334b74e130584c89983 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:59.887928 22435 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:58:59.893661 25266 raft_consensus.cc:1261] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:59.893797 22847 raft_consensus.cc:1261] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:58:59.894096 31342 consensus_queue.cc:1035] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:59.894223 30685 consensus_queue.cc:1035] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:59.897295 22436 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet 1f586c18b98b432ebbaadfa2a277a5cc (table rebalancer_dynamic_table [id=ab63bb39038d4b8bb4622d79cfcb9da3]). Expected version 1 got 0
I20240502 07:58:59.897712 22435 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet 1f586c18b98b432ebbaadfa2a277a5cc (table rebalancer_dynamic_table [id=ab63bb39038d4b8bb4622d79cfcb9da3]). Expected version 1 got 0
I20240502 07:58:59.903888 31606 tablet.cc:1722] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:59.904209 31606 tablet.cc:1620] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:58:59.904618 31476 tablet.cc:1722] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:59.904951 31476 tablet.cc:1620] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:58:59.906831 22436 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet 1f586c18b98b432ebbaadfa2a277a5cc (table rebalancer_dynamic_table [id=ab63bb39038d4b8bb4622d79cfcb9da3]). Expected version 1 got 0
I20240502 07:58:59.908181 22436 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet 1f586c18b98b432ebbaadfa2a277a5cc (table rebalancer_dynamic_table [id=ab63bb39038d4b8bb4622d79cfcb9da3]). Expected version 1 got 0
I20240502 07:58:59.909168 22435 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet 1f586c18b98b432ebbaadfa2a277a5cc (table rebalancer_dynamic_table [id=ab63bb39038d4b8bb4622d79cfcb9da3]). Expected version 1 got 0
I20240502 07:58:59.912956 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=ab63bb39038d4b8bb4622d79cfcb9da3] alter complete (version 1)
W20240502 07:58:59.913818 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:58:59.916800 31607 tablet.cc:1722] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:58:59.917071 31607 tablet.cc:1620] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
W20240502 07:58:59.921576 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:58:59.951408 31027 consensus_queue.cc:1035] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:58:59.954020 31027 consensus_queue.cc:1035] T 073fa19317844cc4aedaf386261268f7 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:58:59.986088 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 78 similar messages]
W20240502 07:59:00.004241 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:59:00.015873 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:59:00.021147 31606 tablet.cc:1722] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:00.021440 31606 tablet.cc:1620] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:00.022470 31607 tablet.cc:1722] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:00.022723 31607 tablet.cc:1620] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:59:00.022928 31476 tablet.cc:1722] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:00.023200 31476 tablet.cc:1620] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:59:00.026341 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=ab63bb39038d4b8bb4622d79cfcb9da3] alter complete (version 2)
I20240502 07:59:00.045415 22438 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_46"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:59:00.051940 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 34ac6e6ef48345418f3e4b31a2fd20af (DEFAULT_TABLE table=rebalancer_extra_table_46 [id=51d896ab3f19480088eb3aeacbcab8ed]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.052549 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 34ac6e6ef48345418f3e4b31a2fd20af. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.052917 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 34ac6e6ef48345418f3e4b31a2fd20af (DEFAULT_TABLE table=rebalancer_extra_table_46 [id=51d896ab3f19480088eb3aeacbcab8ed]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.052948 25246 tablet_service.cc:1467] Processing CreateTablet for tablet ef6ec16977b449d8a48ae068a931292c (DEFAULT_TABLE table=rebalancer_extra_table_46 [id=51d896ab3f19480088eb3aeacbcab8ed]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.053211 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 34ac6e6ef48345418f3e4b31a2fd20af. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.053165 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 34ac6e6ef48345418f3e4b31a2fd20af (DEFAULT_TABLE table=rebalancer_extra_table_46 [id=51d896ab3f19480088eb3aeacbcab8ed]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.053421 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 34ac6e6ef48345418f3e4b31a2fd20af. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.053433 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ef6ec16977b449d8a48ae068a931292c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.055302 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 00d438542f514129aed1f9dec4f1c337 (DEFAULT_TABLE table=rebalancer_extra_table_46 [id=51d896ab3f19480088eb3aeacbcab8ed]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.055367 31478 tablet_bootstrap.cc:492] T 34ac6e6ef48345418f3e4b31a2fd20af P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:00.055552 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00d438542f514129aed1f9dec4f1c337. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.056255 31478 tablet_bootstrap.cc:654] T 34ac6e6ef48345418f3e4b31a2fd20af P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.056917 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 00d438542f514129aed1f9dec4f1c337 (DEFAULT_TABLE table=rebalancer_extra_table_46 [id=51d896ab3f19480088eb3aeacbcab8ed]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.057155 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00d438542f514129aed1f9dec4f1c337. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.058729 31478 tablet_bootstrap.cc:492] T 34ac6e6ef48345418f3e4b31a2fd20af P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:00.058773 31426 tablet_bootstrap.cc:492] T 00d438542f514129aed1f9dec4f1c337 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:00.058804 31478 ts_tablet_manager.cc:1367] T 34ac6e6ef48345418f3e4b31a2fd20af P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:59:00.059262 22564 tablet_service.cc:1467] Processing CreateTablet for tablet ef6ec16977b449d8a48ae068a931292c (DEFAULT_TABLE table=rebalancer_extra_table_46 [id=51d896ab3f19480088eb3aeacbcab8ed]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.059532 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ef6ec16977b449d8a48ae068a931292c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.059518 31478 raft_consensus.cc:348] T 34ac6e6ef48345418f3e4b31a2fd20af P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.059641 31426 tablet_bootstrap.cc:654] T 00d438542f514129aed1f9dec4f1c337 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.059649 31478 raft_consensus.cc:374] T 34ac6e6ef48345418f3e4b31a2fd20af P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.059679 31478 raft_consensus.cc:729] T 34ac6e6ef48345418f3e4b31a2fd20af P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.059794 31478 consensus_queue.cc:260] T 34ac6e6ef48345418f3e4b31a2fd20af P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.060061 31478 ts_tablet_manager.cc:1398] T 34ac6e6ef48345418f3e4b31a2fd20af P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:00.060340 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 00d438542f514129aed1f9dec4f1c337 (DEFAULT_TABLE table=rebalancer_extra_table_46 [id=51d896ab3f19480088eb3aeacbcab8ed]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.060570 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00d438542f514129aed1f9dec4f1c337. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.060629 31426 tablet_bootstrap.cc:492] T 00d438542f514129aed1f9dec4f1c337 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:00.060848 31426 ts_tablet_manager.cc:1367] T 00d438542f514129aed1f9dec4f1c337 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:00.061265 31426 raft_consensus.cc:348] T 00d438542f514129aed1f9dec4f1c337 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:00.061357 31426 raft_consensus.cc:374] T 00d438542f514129aed1f9dec4f1c337 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.061383 31426 raft_consensus.cc:729] T 00d438542f514129aed1f9dec4f1c337 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.061482 31426 consensus_queue.cc:260] T 00d438542f514129aed1f9dec4f1c337 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:00.061695 31478 tablet_bootstrap.cc:492] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:00.061746 31426 ts_tablet_manager.cc:1398] T 00d438542f514129aed1f9dec4f1c337 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:00.062332 22959 tablet_service.cc:1467] Processing CreateTablet for tablet ef6ec16977b449d8a48ae068a931292c (DEFAULT_TABLE table=rebalancer_extra_table_46 [id=51d896ab3f19480088eb3aeacbcab8ed]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.062563 31478 tablet_bootstrap.cc:654] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.062595 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ef6ec16977b449d8a48ae068a931292c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.063593 31478 tablet_bootstrap.cc:492] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:00.063657 31478 ts_tablet_manager.cc:1367] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:00.064028 31478 raft_consensus.cc:348] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.064110 31478 raft_consensus.cc:374] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.064136 31478 raft_consensus.cc:729] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.064229 31478 consensus_queue.cc:260] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.064469 31560 tablet_bootstrap.cc:492] T ef6ec16977b449d8a48ae068a931292c P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:00.064472 31478 ts_tablet_manager.cc:1398] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:00.064859 31641 tablet_bootstrap.cc:492] T 00d438542f514129aed1f9dec4f1c337 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:00.065429 31560 tablet_bootstrap.cc:654] T ef6ec16977b449d8a48ae068a931292c P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.065762 31641 tablet_bootstrap.cc:654] T 00d438542f514129aed1f9dec4f1c337 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.067260 31640 tablet_bootstrap.cc:492] T ef6ec16977b449d8a48ae068a931292c P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:00.067744 31560 tablet_bootstrap.cc:492] T ef6ec16977b449d8a48ae068a931292c P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:00.067821 31560 ts_tablet_manager.cc:1367] T ef6ec16977b449d8a48ae068a931292c P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:59:00.068179 31640 tablet_bootstrap.cc:654] T ef6ec16977b449d8a48ae068a931292c P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.068228 31560 raft_consensus.cc:348] T ef6ec16977b449d8a48ae068a931292c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.068316 31560 raft_consensus.cc:374] T ef6ec16977b449d8a48ae068a931292c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.068349 31560 raft_consensus.cc:729] T ef6ec16977b449d8a48ae068a931292c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.068450 31560 consensus_queue.cc:260] T ef6ec16977b449d8a48ae068a931292c P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.069322 31641 tablet_bootstrap.cc:492] T 00d438542f514129aed1f9dec4f1c337 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:00.069432 31641 ts_tablet_manager.cc:1367] T 00d438542f514129aed1f9dec4f1c337 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:59:00.069444 31643 tablet_bootstrap.cc:492] T 34ac6e6ef48345418f3e4b31a2fd20af P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:00.070214 31560 ts_tablet_manager.cc:1398] T ef6ec16977b449d8a48ae068a931292c P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:00.070443 31643 tablet_bootstrap.cc:654] T 34ac6e6ef48345418f3e4b31a2fd20af P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.070998 31641 raft_consensus.cc:348] T 00d438542f514129aed1f9dec4f1c337 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:00.071128 31641 raft_consensus.cc:374] T 00d438542f514129aed1f9dec4f1c337 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.071170 31641 raft_consensus.cc:729] T 00d438542f514129aed1f9dec4f1c337 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.071288 31641 consensus_queue.cc:260] T 00d438542f514129aed1f9dec4f1c337 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:00.071559 31641 ts_tablet_manager.cc:1398] T 00d438542f514129aed1f9dec4f1c337 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:00.071709 31643 tablet_bootstrap.cc:492] T 34ac6e6ef48345418f3e4b31a2fd20af P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:00.071820 31643 ts_tablet_manager.cc:1367] T 34ac6e6ef48345418f3e4b31a2fd20af P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:59:00.072299 31643 raft_consensus.cc:348] T 34ac6e6ef48345418f3e4b31a2fd20af P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.072440 31643 raft_consensus.cc:374] T 34ac6e6ef48345418f3e4b31a2fd20af P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.072692 31643 raft_consensus.cc:729] T 34ac6e6ef48345418f3e4b31a2fd20af P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.072849 31640 tablet_bootstrap.cc:492] T ef6ec16977b449d8a48ae068a931292c P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:00.072870 31643 consensus_queue.cc:260] T 34ac6e6ef48345418f3e4b31a2fd20af P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.072969 31640 ts_tablet_manager.cc:1367] T ef6ec16977b449d8a48ae068a931292c P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:59:00.073180 31643 ts_tablet_manager.cc:1398] T 34ac6e6ef48345418f3e4b31a2fd20af P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:00.073406 31640 raft_consensus.cc:348] T ef6ec16977b449d8a48ae068a931292c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.071117 31632 tablet_bootstrap.cc:492] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:00.073640 31640 raft_consensus.cc:374] T ef6ec16977b449d8a48ae068a931292c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.074352 31640 raft_consensus.cc:729] T ef6ec16977b449d8a48ae068a931292c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.074545 31640 consensus_queue.cc:260] T ef6ec16977b449d8a48ae068a931292c P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.076261 31640 ts_tablet_manager.cc:1398] T ef6ec16977b449d8a48ae068a931292c P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:59:00.076915 31632 tablet_bootstrap.cc:654] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.078583 31632 tablet_bootstrap.cc:492] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:00.078718 31632 ts_tablet_manager.cc:1367] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.000s
I20240502 07:59:00.079243 31632 raft_consensus.cc:348] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.079357 31632 raft_consensus.cc:374] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.079388 31632 raft_consensus.cc:729] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.079548 31632 consensus_queue.cc:260] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.079808 31632 ts_tablet_manager.cc:1398] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:00.080498 31632 tablet_bootstrap.cc:492] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:00.081305 31632 tablet_bootstrap.cc:654] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.082201 31632 tablet_bootstrap.cc:492] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:00.082317 31632 ts_tablet_manager.cc:1367] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:00.082754 31632 raft_consensus.cc:348] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:00.082876 31632 raft_consensus.cc:374] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.082947 31632 raft_consensus.cc:729] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.083060 31632 consensus_queue.cc:260] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:00.083338 31632 ts_tablet_manager.cc:1398] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:00.091864 31570 raft_consensus.cc:482] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:00.091957 31570 raft_consensus.cc:504] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.092303 31570 leader_election.cc:288] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:00.092587 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ef6ec16977b449d8a48ae068a931292c" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:00.092587 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ef6ec16977b449d8a48ae068a931292c" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:00.092718 22979 raft_consensus.cc:2454] T ef6ec16977b449d8a48ae068a931292c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:59:00.092718 22584 raft_consensus.cc:2454] T ef6ec16977b449d8a48ae068a931292c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:59:00.092967 25202 leader_election.cc:302] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7eced220cf1642f3873fc385b6191e69, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:00.093119 31570 raft_consensus.cc:2781] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:00.093174 31570 raft_consensus.cc:482] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:00.093212 31570 raft_consensus.cc:3037] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.094058 31570 raft_consensus.cc:504] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.094393 31570 leader_election.cc:288] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:00.094666 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ef6ec16977b449d8a48ae068a931292c" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:00.094704 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ef6ec16977b449d8a48ae068a931292c" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:00.094765 22979 raft_consensus.cc:3037] T ef6ec16977b449d8a48ae068a931292c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.094795 22584 raft_consensus.cc:3037] T ef6ec16977b449d8a48ae068a931292c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.095489 22979 raft_consensus.cc:2454] T ef6ec16977b449d8a48ae068a931292c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:59:00.095536 22584 raft_consensus.cc:2454] T ef6ec16977b449d8a48ae068a931292c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:59:00.095764 25200 leader_election.cc:302] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:00.095928 31570 raft_consensus.cc:2781] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:00.095999 31570 raft_consensus.cc:686] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:59:00.096122 31570 consensus_queue.cc:237] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
W20240502 07:59:00.096972 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:59:00.098310 22438 catalog_manager.cc:5484] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
W20240502 07:59:00.102654 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 07:59:00.107457 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
I20240502 07:59:00.109028 31525 raft_consensus.cc:482] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:00.109109 31525 raft_consensus.cc:504] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.109385 31525 leader_election.cc:288] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:00.109652 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "34ac6e6ef48345418f3e4b31a2fd20af" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:00.109673 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "34ac6e6ef48345418f3e4b31a2fd20af" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:00.109774 25265 raft_consensus.cc:2454] T 34ac6e6ef48345418f3e4b31a2fd20af P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:00.109786 22717 raft_consensus.cc:2454] T 34ac6e6ef48345418f3e4b31a2fd20af P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:00.109997 24807 leader_election.cc:302] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
W20240502 07:59:00.110064 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
I20240502 07:59:00.110229 31525 raft_consensus.cc:2781] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:00.110284 31525 raft_consensus.cc:482] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:00.110332 31525 raft_consensus.cc:3037] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.111167 31525 raft_consensus.cc:504] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.111455 31525 leader_election.cc:288] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:00.111753 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "34ac6e6ef48345418f3e4b31a2fd20af" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:00.111750 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "34ac6e6ef48345418f3e4b31a2fd20af" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:00.111836 25265 raft_consensus.cc:3037] T 34ac6e6ef48345418f3e4b31a2fd20af P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.111848 22717 raft_consensus.cc:3037] T 34ac6e6ef48345418f3e4b31a2fd20af P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.112511 25265 raft_consensus.cc:2454] T 34ac6e6ef48345418f3e4b31a2fd20af P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:00.112560 22717 raft_consensus.cc:2454] T 34ac6e6ef48345418f3e4b31a2fd20af P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:00.112804 24807 leader_election.cc:302] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:00.112960 31525 raft_consensus.cc:2781] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:00.113034 31525 raft_consensus.cc:686] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:59:00.113142 31525 consensus_queue.cc:237] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.114867 22438 catalog_manager.cc:5484] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:00.122539 31525 raft_consensus.cc:482] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:00.122622 31525 raft_consensus.cc:504] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:00.122917 31525 leader_election.cc:288] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:00.123208 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00d438542f514129aed1f9dec4f1c337" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:00.123239 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00d438542f514129aed1f9dec4f1c337" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:00.123337 22845 raft_consensus.cc:2454] T 00d438542f514129aed1f9dec4f1c337 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:00.123337 25135 raft_consensus.cc:2454] T 00d438542f514129aed1f9dec4f1c337 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:00.123569 24808 leader_election.cc:302] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:00.123711 31525 raft_consensus.cc:2781] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:00.123760 31525 raft_consensus.cc:482] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:00.123803 31525 raft_consensus.cc:3037] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.124702 31525 raft_consensus.cc:504] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:00.125180 31525 leader_election.cc:288] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:00.125404 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00d438542f514129aed1f9dec4f1c337" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:00.125540 25135 raft_consensus.cc:3037] T 00d438542f514129aed1f9dec4f1c337 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.125583 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00d438542f514129aed1f9dec4f1c337" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:59:00.125692 22845 raft_consensus.cc:3037] T 00d438542f514129aed1f9dec4f1c337 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.126425 25135 raft_consensus.cc:2454] T 00d438542f514129aed1f9dec4f1c337 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:00.126472 22845 raft_consensus.cc:2454] T 00d438542f514129aed1f9dec4f1c337 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:00.126726 24808 leader_election.cc:302] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:00.126853 31525 raft_consensus.cc:2781] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:00.126928 31525 raft_consensus.cc:686] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:59:00.127044 31525 consensus_queue.cc:237] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:00.129001 22438 catalog_manager.cc:5484] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:00.129526 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:59:00.134060 31606 tablet.cc:1722] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:00.134552 31607 tablet.cc:1722] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:00.135478 31476 tablet.cc:1722] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:00.139554 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=ab63bb39038d4b8bb4622d79cfcb9da3] alter complete (version 3)
W20240502 07:59:00.182093 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 07:59:00.186748 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:00.239714 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:59:00.240988 22436 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:00.241119 22436 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:00.243095 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 1f586c18b98b432ebbaadfa2a277a5cc
I20240502 07:59:00.243695 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet 1f586c18b98b432ebbaadfa2a277a5cc with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:00 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:59:00.243707 22828 tablet_service.cc:1514] Processing DeleteTablet for tablet 1f586c18b98b432ebbaadfa2a277a5cc with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:00 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:59:00.243707 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet 1f586c18b98b432ebbaadfa2a277a5cc with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:00 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:59:00.243990 31660 tablet_replica.cc:330] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:59:00.243988 31658 tablet_replica.cc:330] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:59:00.244100 31658 raft_consensus.cc:2229] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:59:00.244127 31660 raft_consensus.cc:2229] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:00.244244 31660 raft_consensus.cc:2258] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:00.244294 31658 raft_consensus.cc:2258] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:00.244547 31659 tablet_replica.cc:330] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:59:00.244791 31659 raft_consensus.cc:2229] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:00.244874 31660 ts_tablet_manager.cc:1875] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:00.244885 31658 ts_tablet_manager.cc:1875] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:00.244985 31659 raft_consensus.cc:2258] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:00.245501 31659 ts_tablet_manager.cc:1875] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:00.247207 31659 ts_tablet_manager.cc:1888] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:00.247225 31658 ts_tablet_manager.cc:1888] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:00.247272 31659 log.cc:1196] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/1f586c18b98b432ebbaadfa2a277a5cc
I20240502 07:59:00.247285 31658 log.cc:1196] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/1f586c18b98b432ebbaadfa2a277a5cc
I20240502 07:59:00.247274 31660 ts_tablet_manager.cc:1888] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:00.247328 31660 log.cc:1196] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/1f586c18b98b432ebbaadfa2a277a5cc
I20240502 07:59:00.247592 31659 ts_tablet_manager.cc:1909] T 1f586c18b98b432ebbaadfa2a277a5cc P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:59:00.247630 31660 ts_tablet_manager.cc:1909] T 1f586c18b98b432ebbaadfa2a277a5cc P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:59:00.247630 31658 ts_tablet_manager.cc:1909] T 1f586c18b98b432ebbaadfa2a277a5cc P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:59:00.248239 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 1f586c18b98b432ebbaadfa2a277a5cc (table rebalancer_dynamic_table_new_name [id=ab63bb39038d4b8bb4622d79cfcb9da3]) successfully deleted
I20240502 07:59:00.248353 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 1f586c18b98b432ebbaadfa2a277a5cc (table rebalancer_dynamic_table_new_name [id=ab63bb39038d4b8bb4622d79cfcb9da3]) successfully deleted
I20240502 07:59:00.248394 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet 1f586c18b98b432ebbaadfa2a277a5cc (table rebalancer_dynamic_table_new_name [id=ab63bb39038d4b8bb4622d79cfcb9da3]) successfully deleted
W20240502 07:59:00.254099 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20240502 07:59:00.269330 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 07:59:00.276521 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 07:59:00.286456 22430 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 14 similar messages]
W20240502 07:59:00.287159 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_DELETED RPC failed for tablet 41284d865ebb4cd18f2ce08f66343262: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:00.304785 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 07:59:00.316104 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:59:00.344779 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:59:00.350004 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 2e24d00f7d2f4b119091e5cc2b5370c5 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=05f81533af7b42dd9ec98badb671f7e2]), partition=
I20240502 07:59:00.350231 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 2e24d00f7d2f4b119091e5cc2b5370c5 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=05f81533af7b42dd9ec98badb671f7e2]), partition=
I20240502 07:59:00.350275 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e24d00f7d2f4b119091e5cc2b5370c5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.350458 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e24d00f7d2f4b119091e5cc2b5370c5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.350435 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 2e24d00f7d2f4b119091e5cc2b5370c5 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=05f81533af7b42dd9ec98badb671f7e2]), partition=
I20240502 07:59:00.350656 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e24d00f7d2f4b119091e5cc2b5370c5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.352572 31640 tablet_bootstrap.cc:492] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:00.352733 31560 tablet_bootstrap.cc:492] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:00.353331 31640 tablet_bootstrap.cc:654] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.353458 31560 tablet_bootstrap.cc:654] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.353788 31643 tablet_bootstrap.cc:492] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:00.354404 31560 tablet_bootstrap.cc:492] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:00.354466 31643 tablet_bootstrap.cc:654] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.354475 31560 ts_tablet_manager.cc:1367] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:00.354982 31560 raft_consensus.cc:348] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:00.355105 31560 raft_consensus.cc:374] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.355137 31560 raft_consensus.cc:729] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.355247 31560 consensus_queue.cc:260] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:00.355554 31560 ts_tablet_manager.cc:1398] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:00.356696 31643 tablet_bootstrap.cc:492] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:00.356781 31643 ts_tablet_manager.cc:1367] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:59:00.357110 31643 raft_consensus.cc:348] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:00.357185 31643 raft_consensus.cc:374] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.357204 31643 raft_consensus.cc:729] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.357311 31643 consensus_queue.cc:260] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:00.357558 31643 ts_tablet_manager.cc:1398] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:00.359086 31640 tablet_bootstrap.cc:492] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:00.359175 31640 ts_tablet_manager.cc:1367] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 07:59:00.359633 31640 raft_consensus.cc:348] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:00.359723 31640 raft_consensus.cc:374] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.359753 31640 raft_consensus.cc:729] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.359874 31640 consensus_queue.cc:260] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:00.360112 31640 ts_tablet_manager.cc:1398] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
W20240502 07:59:00.371786 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 07:59:00.398027 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 07:59:00.417137 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 07:59:00.492213 31570 consensus_queue.cc:1035] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:00.494912 31698 consensus_queue.cc:1035] T ef6ec16977b449d8a48ae068a931292c P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:00.522610 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:59:00.541376 31696 consensus_queue.cc:1035] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:00.544873 31358 consensus_queue.cc:1035] T 00d438542f514129aed1f9dec4f1c337 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:00.556530 31696 consensus_queue.cc:1035] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:00.559638 31358 consensus_queue.cc:1035] T 34ac6e6ef48345418f3e4b31a2fd20af P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:00.591280 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:00.638702 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_48"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:59:00.644004 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 06510e3fbedf4e89b64273d41671c039 (DEFAULT_TABLE table=rebalancer_extra_table_48 [id=9c3685fd8b3f4df6b0cf355b32bf600b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.644063 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 06510e3fbedf4e89b64273d41671c039 (DEFAULT_TABLE table=rebalancer_extra_table_48 [id=9c3685fd8b3f4df6b0cf355b32bf600b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.644353 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 06510e3fbedf4e89b64273d41671c039. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.644394 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 06510e3fbedf4e89b64273d41671c039. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.644786 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 06510e3fbedf4e89b64273d41671c039 (DEFAULT_TABLE table=rebalancer_extra_table_48 [id=9c3685fd8b3f4df6b0cf355b32bf600b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.645026 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 06510e3fbedf4e89b64273d41671c039. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.646389 31738 tablet_bootstrap.cc:492] T 06510e3fbedf4e89b64273d41671c039 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:00.646548 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 9008dd5678d8495481e1b66212dc0899 (DEFAULT_TABLE table=rebalancer_extra_table_48 [id=9c3685fd8b3f4df6b0cf355b32bf600b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.646785 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9008dd5678d8495481e1b66212dc0899. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.646741 24850 tablet_service.cc:1467] Processing CreateTablet for tablet ac0e1534816244b1838b022b109a4a7e (DEFAULT_TABLE table=rebalancer_extra_table_48 [id=9c3685fd8b3f4df6b0cf355b32bf600b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.646951 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ac0e1534816244b1838b022b109a4a7e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.647038 31560 tablet_bootstrap.cc:492] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:00.647172 22957 tablet_service.cc:1467] Processing CreateTablet for tablet ac0e1534816244b1838b022b109a4a7e (DEFAULT_TABLE table=rebalancer_extra_table_48 [id=9c3685fd8b3f4df6b0cf355b32bf600b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.647364 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ac0e1534816244b1838b022b109a4a7e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.647720 31738 tablet_bootstrap.cc:654] T 06510e3fbedf4e89b64273d41671c039 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.647822 31560 tablet_bootstrap.cc:654] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.648648 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 9008dd5678d8495481e1b66212dc0899 (DEFAULT_TABLE table=rebalancer_extra_table_48 [id=9c3685fd8b3f4df6b0cf355b32bf600b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.648901 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9008dd5678d8495481e1b66212dc0899. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.648949 31741 tablet_bootstrap.cc:492] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:00.649009 31643 tablet_bootstrap.cc:492] T 06510e3fbedf4e89b64273d41671c039 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:00.649803 31643 tablet_bootstrap.cc:654] T 06510e3fbedf4e89b64273d41671c039 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.649865 31741 tablet_bootstrap.cc:654] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.651461 22695 tablet_service.cc:1467] Processing CreateTablet for tablet ac0e1534816244b1838b022b109a4a7e (DEFAULT_TABLE table=rebalancer_extra_table_48 [id=9c3685fd8b3f4df6b0cf355b32bf600b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.651676 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ac0e1534816244b1838b022b109a4a7e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.651667 31738 tablet_bootstrap.cc:492] T 06510e3fbedf4e89b64273d41671c039 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:00.651748 31738 ts_tablet_manager.cc:1367] T 06510e3fbedf4e89b64273d41671c039 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:59:00.652279 31738 raft_consensus.cc:348] T 06510e3fbedf4e89b64273d41671c039 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.652478 31738 raft_consensus.cc:374] T 06510e3fbedf4e89b64273d41671c039 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.652570 31738 raft_consensus.cc:729] T 06510e3fbedf4e89b64273d41671c039 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.652772 31738 consensus_queue.cc:260] T 06510e3fbedf4e89b64273d41671c039 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.653493 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 9008dd5678d8495481e1b66212dc0899 (DEFAULT_TABLE table=rebalancer_extra_table_48 [id=9c3685fd8b3f4df6b0cf355b32bf600b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:00.653800 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9008dd5678d8495481e1b66212dc0899. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:00.654300 31643 tablet_bootstrap.cc:492] T 06510e3fbedf4e89b64273d41671c039 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:00.654402 31643 ts_tablet_manager.cc:1367] T 06510e3fbedf4e89b64273d41671c039 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:59:00.654745 31738 ts_tablet_manager.cc:1398] T 06510e3fbedf4e89b64273d41671c039 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.003s user 0.000s sys 0.001s
I20240502 07:59:00.654846 31643 raft_consensus.cc:348] T 06510e3fbedf4e89b64273d41671c039 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.654953 31643 raft_consensus.cc:374] T 06510e3fbedf4e89b64273d41671c039 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.654992 31643 raft_consensus.cc:729] T 06510e3fbedf4e89b64273d41671c039 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.655021 31744 tablet_bootstrap.cc:492] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:00.655107 31643 consensus_queue.cc:260] T 06510e3fbedf4e89b64273d41671c039 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.655462 31643 ts_tablet_manager.cc:1398] T 06510e3fbedf4e89b64273d41671c039 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:00.655654 31643 tablet_bootstrap.cc:492] T ac0e1534816244b1838b022b109a4a7e P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:00.656122 31738 tablet_bootstrap.cc:492] T 9008dd5678d8495481e1b66212dc0899 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:00.656419 31643 tablet_bootstrap.cc:654] T ac0e1534816244b1838b022b109a4a7e P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.657402 31744 tablet_bootstrap.cc:654] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.658105 31356 raft_consensus.cc:482] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:00.658180 31356 raft_consensus.cc:504] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:00.658468 31356 leader_election.cc:288] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:59:00.658730 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e24d00f7d2f4b119091e5cc2b5370c5" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:00.658869 22979 raft_consensus.cc:2454] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:59:00.659090 22651 leader_election.cc:302] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:00.659207 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e24d00f7d2f4b119091e5cc2b5370c5" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:00.659245 31745 raft_consensus.cc:2781] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:00.659296 31745 raft_consensus.cc:482] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:00.659319 22586 raft_consensus.cc:2454] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:59:00.659339 31745 raft_consensus.cc:3037] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.659344 31738 tablet_bootstrap.cc:654] T 9008dd5678d8495481e1b66212dc0899 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
W20240502 07:59:00.659415 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
I20240502 07:59:00.660204 31741 tablet_bootstrap.cc:492] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:00.660333 31741 ts_tablet_manager.cc:1367] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.000s
I20240502 07:59:00.660687 31745 raft_consensus.cc:504] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:00.660993 31741 raft_consensus.cc:348] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.661126 31741 raft_consensus.cc:374] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.661180 31741 raft_consensus.cc:729] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.661312 31741 consensus_queue.cc:260] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.661419 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e24d00f7d2f4b119091e5cc2b5370c5" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:00.661517 22979 raft_consensus.cc:3037] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.661907 31560 tablet_bootstrap.cc:492] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:00.661974 31560 ts_tablet_manager.cc:1367] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.015s user 0.002s sys 0.000s
I20240502 07:59:00.662137 31744 tablet_bootstrap.cc:492] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:00.662209 31744 ts_tablet_manager.cc:1367] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
I20240502 07:59:00.661505 31738 tablet_bootstrap.cc:492] T 9008dd5678d8495481e1b66212dc0899 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:00.662412 31738 ts_tablet_manager.cc:1367] T 9008dd5678d8495481e1b66212dc0899 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.002s
I20240502 07:59:00.662400 22979 raft_consensus.cc:2454] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:59:00.662400 31560 raft_consensus.cc:348] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.662531 31560 raft_consensus.cc:374] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.662559 31560 raft_consensus.cc:729] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.662631 31744 raft_consensus.cc:348] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:00.662685 31560 consensus_queue.cc:260] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.662752 31744 raft_consensus.cc:374] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.662797 31744 raft_consensus.cc:729] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.662905 22651 leader_election.cc:302] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:00.662899 31744 consensus_queue.cc:260] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:00.662899 31738 raft_consensus.cc:348] T 9008dd5678d8495481e1b66212dc0899 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:00.662964 31560 ts_tablet_manager.cc:1398] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:00.662976 31738 raft_consensus.cc:374] T 9008dd5678d8495481e1b66212dc0899 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.663010 31738 raft_consensus.cc:729] T 9008dd5678d8495481e1b66212dc0899 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.663116 31560 tablet_bootstrap.cc:492] T ac0e1534816244b1838b022b109a4a7e P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:00.663116 31356 raft_consensus.cc:2781] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:00.663158 31744 ts_tablet_manager.cc:1398] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:00.663123 31738 consensus_queue.cc:260] T 9008dd5678d8495481e1b66212dc0899 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:00.663276 31356 raft_consensus.cc:686] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:59:00.663365 31738 ts_tablet_manager.cc:1398] T 9008dd5678d8495481e1b66212dc0899 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:00.663389 31356 consensus_queue.cc:237] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:00.664026 31560 tablet_bootstrap.cc:654] T ac0e1534816244b1838b022b109a4a7e P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.666132 31741 ts_tablet_manager.cc:1398] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:59:00.667006 22438 catalog_manager.cc:5484] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:00.668246 31741 tablet_bootstrap.cc:492] T 9008dd5678d8495481e1b66212dc0899 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:00.668249 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e24d00f7d2f4b119091e5cc2b5370c5" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:00.668360 22586 raft_consensus.cc:3037] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.669378 31741 tablet_bootstrap.cc:654] T 9008dd5678d8495481e1b66212dc0899 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:00.669376 22586 raft_consensus.cc:2454] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:59:00.670363 31745 leader_election.cc:288] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:59:00.670567 31741 tablet_bootstrap.cc:492] T 9008dd5678d8495481e1b66212dc0899 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:00.670643 31741 ts_tablet_manager.cc:1367] T 9008dd5678d8495481e1b66212dc0899 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:00.670694 31560 tablet_bootstrap.cc:492] T ac0e1534816244b1838b022b109a4a7e P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:00.670819 31560 ts_tablet_manager.cc:1367] T ac0e1534816244b1838b022b109a4a7e P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.000s
I20240502 07:59:00.671135 31741 raft_consensus.cc:348] T 9008dd5678d8495481e1b66212dc0899 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:00.671231 31741 raft_consensus.cc:374] T 9008dd5678d8495481e1b66212dc0899 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.671272 31741 raft_consensus.cc:729] T 9008dd5678d8495481e1b66212dc0899 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.671241 31560 raft_consensus.cc:348] T ac0e1534816244b1838b022b109a4a7e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.671329 31560 raft_consensus.cc:374] T ac0e1534816244b1838b022b109a4a7e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.671371 31560 raft_consensus.cc:729] T ac0e1534816244b1838b022b109a4a7e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.671388 31741 consensus_queue.cc:260] T 9008dd5678d8495481e1b66212dc0899 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:00.671504 31560 consensus_queue.cc:260] T ac0e1534816244b1838b022b109a4a7e P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.671666 31741 ts_tablet_manager.cc:1398] T 9008dd5678d8495481e1b66212dc0899 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:00.671770 31560 ts_tablet_manager.cc:1398] T ac0e1534816244b1838b022b109a4a7e P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:00.673683 31643 tablet_bootstrap.cc:492] T ac0e1534816244b1838b022b109a4a7e P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:00.673771 31643 ts_tablet_manager.cc:1367] T ac0e1534816244b1838b022b109a4a7e P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.018s user 0.001s sys 0.000s
I20240502 07:59:00.674248 31643 raft_consensus.cc:348] T ac0e1534816244b1838b022b109a4a7e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.674386 31643 raft_consensus.cc:374] T ac0e1534816244b1838b022b109a4a7e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:00.674418 31643 raft_consensus.cc:729] T ac0e1534816244b1838b022b109a4a7e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:00.674556 31643 consensus_queue.cc:260] T ac0e1534816244b1838b022b109a4a7e P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.674814 31643 ts_tablet_manager.cc:1398] T ac0e1534816244b1838b022b109a4a7e P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:00.679754 31358 raft_consensus.cc:482] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:00.679827 31358 raft_consensus.cc:504] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.680114 31358 leader_election.cc:288] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:00.680385 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ac0e1534816244b1838b022b109a4a7e" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:00.680383 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ac0e1534816244b1838b022b109a4a7e" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:00.680481 22979 raft_consensus.cc:2454] T ac0e1534816244b1838b022b109a4a7e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:00.680485 22716 raft_consensus.cc:2454] T ac0e1534816244b1838b022b109a4a7e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:00.680706 24806 leader_election.cc:302] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:00.680869 31358 raft_consensus.cc:2781] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:00.680932 31358 raft_consensus.cc:482] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:00.680964 31358 raft_consensus.cc:3037] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.681628 31358 raft_consensus.cc:504] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.681921 31358 leader_election.cc:288] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:00.682154 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ac0e1534816244b1838b022b109a4a7e" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:00.682252 22979 raft_consensus.cc:3037] T ac0e1534816244b1838b022b109a4a7e P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.682294 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ac0e1534816244b1838b022b109a4a7e" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:00.682375 22716 raft_consensus.cc:3037] T ac0e1534816244b1838b022b109a4a7e P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.683121 22979 raft_consensus.cc:2454] T ac0e1534816244b1838b022b109a4a7e P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:00.683240 22716 raft_consensus.cc:2454] T ac0e1534816244b1838b022b109a4a7e P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:00.683444 24806 leader_election.cc:302] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:00.683588 31358 raft_consensus.cc:2781] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:00.683681 31358 raft_consensus.cc:686] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:59:00.683830 31358 consensus_queue.cc:237] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:00.685750 22444 catalog_manager.cc:5484] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:00.695045 31609 raft_consensus.cc:482] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:00.695143 31609 raft_consensus.cc:504] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:00.695685 31609 leader_election.cc:288] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:59:00.696012 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9008dd5678d8495481e1b66212dc0899" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:00.696002 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9008dd5678d8495481e1b66212dc0899" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:00.696143 24872 raft_consensus.cc:2454] T 9008dd5678d8495481e1b66212dc0899 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:00.696144 25133 raft_consensus.cc:2454] T 9008dd5678d8495481e1b66212dc0899 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:00.696444 22781 leader_election.cc:302] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:00.696677 31609 raft_consensus.cc:2781] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:00.696741 31609 raft_consensus.cc:482] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:00.696782 31609 raft_consensus.cc:3037] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.697569 31609 raft_consensus.cc:504] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:00.697854 31609 leader_election.cc:288] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:59:00.698143 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9008dd5678d8495481e1b66212dc0899" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:00.698148 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9008dd5678d8495481e1b66212dc0899" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:00.698227 24872 raft_consensus.cc:3037] T 9008dd5678d8495481e1b66212dc0899 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.698242 25133 raft_consensus.cc:3037] T 9008dd5678d8495481e1b66212dc0899 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.698905 24872 raft_consensus.cc:2454] T 9008dd5678d8495481e1b66212dc0899 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:00.699025 25133 raft_consensus.cc:2454] T 9008dd5678d8495481e1b66212dc0899 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:00.699164 22781 leader_election.cc:302] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:00.699293 31609 raft_consensus.cc:2781] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:00.699378 31609 raft_consensus.cc:686] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:59:00.699523 31609 consensus_queue.cc:237] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:00.701550 22438 catalog_manager.cc:5484] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:00.707782 30685 raft_consensus.cc:482] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:00.707866 30685 raft_consensus.cc:504] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.708207 30685 leader_election.cc:288] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:00.708515 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "06510e3fbedf4e89b64273d41671c039" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:00.708642 22716 raft_consensus.cc:2454] T 06510e3fbedf4e89b64273d41671c039 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:59:00.708691 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "06510e3fbedf4e89b64273d41671c039" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:00.708792 25133 raft_consensus.cc:2454] T 06510e3fbedf4e89b64273d41671c039 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:59:00.708977 22912 leader_election.cc:302] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:00.709141 30685 raft_consensus.cc:2781] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:00.709192 30685 raft_consensus.cc:482] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:00.709229 30685 raft_consensus.cc:3037] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.709901 30685 raft_consensus.cc:504] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.710191 30685 leader_election.cc:288] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:00.710458 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "06510e3fbedf4e89b64273d41671c039" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:00.710466 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "06510e3fbedf4e89b64273d41671c039" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:00.710548 25133 raft_consensus.cc:3037] T 06510e3fbedf4e89b64273d41671c039 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.710572 22716 raft_consensus.cc:3037] T 06510e3fbedf4e89b64273d41671c039 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:00.711268 22716 raft_consensus.cc:2454] T 06510e3fbedf4e89b64273d41671c039 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:59:00.711383 25133 raft_consensus.cc:2454] T 06510e3fbedf4e89b64273d41671c039 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:59:00.711513 22912 leader_election.cc:302] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:00.711671 30685 raft_consensus.cc:2781] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:00.711787 30685 raft_consensus.cc:686] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:59:00.711951 30685 consensus_queue.cc:237] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:00.714005 22444 catalog_manager.cc:5484] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } }
I20240502 07:59:00.809808 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:59:00.814026 22586 raft_consensus.cc:1261] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:00.814011 22979 raft_consensus.cc:1261] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:00.814504 31356 consensus_queue.cc:1035] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:00.814831 31745 consensus_queue.cc:1035] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:00.817642 31790 tablet.cc:1722] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:00.817760 22444 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet 2e24d00f7d2f4b119091e5cc2b5370c5 (table rebalancer_dynamic_table [id=05f81533af7b42dd9ec98badb671f7e2]). Expected version 1 got 0
I20240502 07:59:00.818015 31790 tablet.cc:1620] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:59:00.818497 22438 catalog_manager.cc:5576] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451) does not have the latest schema for tablet 2e24d00f7d2f4b119091e5cc2b5370c5 (table rebalancer_dynamic_table [id=05f81533af7b42dd9ec98badb671f7e2]). Expected version 1 got 0
I20240502 07:59:00.819731 31792 tablet.cc:1722] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:00.820061 31792 tablet.cc:1620] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:00.820899 22438 catalog_manager.cc:5576] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451) does not have the latest schema for tablet 2e24d00f7d2f4b119091e5cc2b5370c5 (table rebalancer_dynamic_table [id=05f81533af7b42dd9ec98badb671f7e2]). Expected version 1 got 0
I20240502 07:59:00.821604 31791 tablet.cc:1722] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:00.821872 31791 tablet.cc:1620] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:59:00.821911 22444 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet 2e24d00f7d2f4b119091e5cc2b5370c5 (table rebalancer_dynamic_table [id=05f81533af7b42dd9ec98badb671f7e2]). Expected version 1 got 0
I20240502 07:59:00.823434 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=05f81533af7b42dd9ec98badb671f7e2] alter complete (version 1)
W20240502 07:59:00.837764 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:59:00.924803 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:59:00.929222 31790 tablet.cc:1722] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:00.929579 31790 tablet.cc:1620] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:59:00.929766 31792 tablet.cc:1722] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:00.930050 31792 tablet.cc:1620] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:00.930315 31791 tablet.cc:1722] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:00.930873 31791 tablet.cc:1620] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:59:00.935246 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=05f81533af7b42dd9ec98badb671f7e2] alter complete (version 2)
W20240502 07:59:01.027750 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:59:01.037223 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:59:01.041767 31790 tablet.cc:1722] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:01.042516 31792 tablet.cc:1722] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:01.042979 31791 tablet.cc:1722] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:01.047664 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=05f81533af7b42dd9ec98badb671f7e2] alter complete (version 3)
W20240502 07:59:01.068755 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 07:59:01.071624 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:01.110217 30685 consensus_queue.cc:1035] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:01.113253 30685 consensus_queue.cc:1035] T 06510e3fbedf4e89b64273d41671c039 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:01.142948 31525 consensus_queue.cc:1035] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:01.146497 31358 consensus_queue.cc:1035] T ac0e1534816244b1838b022b109a4a7e P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:01.147989 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:59:01.150137 22436 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:01.150250 22436 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:01.152149 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 2e24d00f7d2f4b119091e5cc2b5370c5
I20240502 07:59:01.152549 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet 2e24d00f7d2f4b119091e5cc2b5370c5 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:01 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:59:01.152644 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet 2e24d00f7d2f4b119091e5cc2b5370c5 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:01 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:59:01.152899 31830 tablet_replica.cc:330] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:59:01.152971 31831 tablet_replica.cc:330] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:59:01.152956 22564 tablet_service.cc:1514] Processing DeleteTablet for tablet 2e24d00f7d2f4b119091e5cc2b5370c5 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:01 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:59:01.153066 31830 raft_consensus.cc:2229] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:59:01.153106 31831 raft_consensus.cc:2229] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:01.153216 31832 tablet_replica.cc:330] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:59:01.153254 31831 raft_consensus.cc:2258] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:01.153373 31832 raft_consensus.cc:2229] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:01.153406 31830 raft_consensus.cc:2258] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:01.153515 31832 raft_consensus.cc:2258] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:01.154167 31832 ts_tablet_manager.cc:1875] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:01.154285 31830 ts_tablet_manager.cc:1875] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:01.155045 31831 ts_tablet_manager.cc:1875] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:01.156265 31832 ts_tablet_manager.cc:1888] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:01.156368 31832 log.cc:1196] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/2e24d00f7d2f4b119091e5cc2b5370c5
I20240502 07:59:01.156759 31832 ts_tablet_manager.cc:1909] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:59:01.156930 31830 ts_tablet_manager.cc:1888] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:01.157001 31830 log.cc:1196] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/2e24d00f7d2f4b119091e5cc2b5370c5
I20240502 07:59:01.157320 31830 ts_tablet_manager.cc:1909] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:59:01.157383 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 2e24d00f7d2f4b119091e5cc2b5370c5 (table rebalancer_dynamic_table_new_name [id=05f81533af7b42dd9ec98badb671f7e2]) successfully deleted
I20240502 07:59:01.157860 31831 ts_tablet_manager.cc:1888] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:01.157958 31831 log.cc:1196] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/2e24d00f7d2f4b119091e5cc2b5370c5
I20240502 07:59:01.157930 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 2e24d00f7d2f4b119091e5cc2b5370c5 (table rebalancer_dynamic_table_new_name [id=05f81533af7b42dd9ec98badb671f7e2]) successfully deleted
I20240502 07:59:01.158244 31831 ts_tablet_manager.cc:1909] T 2e24d00f7d2f4b119091e5cc2b5370c5 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:59:01.158732 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 2e24d00f7d2f4b119091e5cc2b5370c5 (table rebalancer_dynamic_table_new_name [id=05f81533af7b42dd9ec98badb671f7e2]) successfully deleted
W20240502 07:59:01.164115 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 07:59:01.175652 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:59:01.230134 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_50"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:59:01.236380 25246 tablet_service.cc:1467] Processing CreateTablet for tablet eb4abf170cb9477aad245d46676773b4 (DEFAULT_TABLE table=rebalancer_extra_table_50 [id=69ab8f9519614e859f1b2b4b9761c66e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:01.236858 25235 tablet_service.cc:1467] Processing CreateTablet for tablet e85e36bb62604473be3b9a06bfd24a66 (DEFAULT_TABLE table=rebalancer_extra_table_50 [id=69ab8f9519614e859f1b2b4b9761c66e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:01.237244 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb4abf170cb9477aad245d46676773b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:01.237483 25113 tablet_service.cc:1467] Processing CreateTablet for tablet e85e36bb62604473be3b9a06bfd24a66 (DEFAULT_TABLE table=rebalancer_extra_table_50 [id=69ab8f9519614e859f1b2b4b9761c66e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:01.237478 22566 tablet_service.cc:1467] Processing CreateTablet for tablet e85e36bb62604473be3b9a06bfd24a66 (DEFAULT_TABLE table=rebalancer_extra_table_50 [id=69ab8f9519614e859f1b2b4b9761c66e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:01.237495 25245 tablet_service.cc:1467] Processing CreateTablet for tablet aa32346334994db4ace760fd2f08982f (DEFAULT_TABLE table=rebalancer_extra_table_50 [id=69ab8f9519614e859f1b2b4b9761c66e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:01.237704 25245 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aa32346334994db4ace760fd2f08982f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:01.237743 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e85e36bb62604473be3b9a06bfd24a66. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:01.237743 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e85e36bb62604473be3b9a06bfd24a66. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:01.239704 22957 tablet_service.cc:1467] Processing CreateTablet for tablet eb4abf170cb9477aad245d46676773b4 (DEFAULT_TABLE table=rebalancer_extra_table_50 [id=69ab8f9519614e859f1b2b4b9761c66e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:01.239950 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb4abf170cb9477aad245d46676773b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:01.239931 25115 tablet_service.cc:1467] Processing CreateTablet for tablet aa32346334994db4ace760fd2f08982f (DEFAULT_TABLE table=rebalancer_extra_table_50 [id=69ab8f9519614e859f1b2b4b9761c66e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:01.239991 31855 tablet_bootstrap.cc:492] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:01.240134 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aa32346334994db4ace760fd2f08982f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:01.240960 31854 tablet_bootstrap.cc:492] T e85e36bb62604473be3b9a06bfd24a66 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:01.240998 31855 tablet_bootstrap.cc:654] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:01.241827 31854 tablet_bootstrap.cc:654] T e85e36bb62604473be3b9a06bfd24a66 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:01.242201 24852 tablet_service.cc:1467] Processing CreateTablet for tablet aa32346334994db4ace760fd2f08982f (DEFAULT_TABLE table=rebalancer_extra_table_50 [id=69ab8f9519614e859f1b2b4b9761c66e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:01.242425 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e85e36bb62604473be3b9a06bfd24a66. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:01.242475 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aa32346334994db4ace760fd2f08982f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:01.244611 31859 tablet_bootstrap.cc:492] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:01.244809 22564 tablet_service.cc:1467] Processing CreateTablet for tablet eb4abf170cb9477aad245d46676773b4 (DEFAULT_TABLE table=rebalancer_extra_table_50 [id=69ab8f9519614e859f1b2b4b9761c66e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:01.245035 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb4abf170cb9477aad245d46676773b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:01.245270 31854 tablet_bootstrap.cc:492] T e85e36bb62604473be3b9a06bfd24a66 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:01.245348 31854 ts_tablet_manager.cc:1367] T e85e36bb62604473be3b9a06bfd24a66 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:59:01.245770 31859 tablet_bootstrap.cc:654] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:01.245836 31854 raft_consensus.cc:348] T e85e36bb62604473be3b9a06bfd24a66 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:01.245949 31854 raft_consensus.cc:374] T e85e36bb62604473be3b9a06bfd24a66 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:01.245978 31854 raft_consensus.cc:729] T e85e36bb62604473be3b9a06bfd24a66 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:01.246081 31854 consensus_queue.cc:260] T e85e36bb62604473be3b9a06bfd24a66 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:01.246517 31854 ts_tablet_manager.cc:1398] T e85e36bb62604473be3b9a06bfd24a66 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:01.247654 31854 tablet_bootstrap.cc:492] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:01.248584 31859 tablet_bootstrap.cc:492] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:01.248747 31859 ts_tablet_manager.cc:1367] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:59:01.248879 31854 tablet_bootstrap.cc:654] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:01.249296 31859 raft_consensus.cc:348] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:01.249454 31853 tablet_bootstrap.cc:492] T aa32346334994db4ace760fd2f08982f P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:01.249464 31859 raft_consensus.cc:374] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:01.249569 31859 raft_consensus.cc:729] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:01.249733 31859 consensus_queue.cc:260] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:01.249799 31854 tablet_bootstrap.cc:492] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:01.249845 31857 tablet_bootstrap.cc:492] T eb4abf170cb9477aad245d46676773b4 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:01.249894 31854 ts_tablet_manager.cc:1367] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:01.250208 31859 ts_tablet_manager.cc:1398] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:01.250800 31857 tablet_bootstrap.cc:654] T eb4abf170cb9477aad245d46676773b4 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:01.251914 31853 tablet_bootstrap.cc:654] T aa32346334994db4ace760fd2f08982f P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:01.251895 31854 raft_consensus.cc:348] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:01.251986 31854 raft_consensus.cc:374] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:01.252022 31854 raft_consensus.cc:729] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:01.252127 31854 consensus_queue.cc:260] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:01.252367 31854 ts_tablet_manager.cc:1398] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 07:59:01.253146 31853 tablet_bootstrap.cc:492] T aa32346334994db4ace760fd2f08982f P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:01.253232 31853 ts_tablet_manager.cc:1367] T aa32346334994db4ace760fd2f08982f P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:59:01.253341 31609 consensus_queue.cc:1035] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:01.253636 31853 raft_consensus.cc:348] T aa32346334994db4ace760fd2f08982f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:01.253759 31853 raft_consensus.cc:374] T aa32346334994db4ace760fd2f08982f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:01.253808 31853 raft_consensus.cc:729] T aa32346334994db4ace760fd2f08982f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:01.253929 31853 consensus_queue.cc:260] T aa32346334994db4ace760fd2f08982f P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:01.254217 31853 ts_tablet_manager.cc:1398] T aa32346334994db4ace760fd2f08982f P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:01.255082 31853 tablet_bootstrap.cc:492] T e85e36bb62604473be3b9a06bfd24a66 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:01.255970 31853 tablet_bootstrap.cc:654] T e85e36bb62604473be3b9a06bfd24a66 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:01.257459 31853 tablet_bootstrap.cc:492] T e85e36bb62604473be3b9a06bfd24a66 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:01.257467 31855 tablet_bootstrap.cc:492] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:01.257550 31855 ts_tablet_manager.cc:1367] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.018s user 0.001s sys 0.000s
I20240502 07:59:01.258088 31855 raft_consensus.cc:348] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:01.258203 31855 raft_consensus.cc:374] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:01.258265 31855 raft_consensus.cc:729] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:01.258368 31855 consensus_queue.cc:260] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:01.258704 31855 ts_tablet_manager.cc:1398] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:01.258993 31855 tablet_bootstrap.cc:492] T aa32346334994db4ace760fd2f08982f P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:01.259611 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:59:01.259790 31855 tablet_bootstrap.cc:654] T aa32346334994db4ace760fd2f08982f P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:01.259864 31853 ts_tablet_manager.cc:1367] T e85e36bb62604473be3b9a06bfd24a66 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:59:01.260411 31853 raft_consensus.cc:348] T e85e36bb62604473be3b9a06bfd24a66 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:01.260527 31853 raft_consensus.cc:374] T e85e36bb62604473be3b9a06bfd24a66 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:01.260576 31853 raft_consensus.cc:729] T e85e36bb62604473be3b9a06bfd24a66 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:01.261319 31853 consensus_queue.cc:260] T e85e36bb62604473be3b9a06bfd24a66 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
W20240502 07:59:01.261628 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:59:01.261657 31853 ts_tablet_manager.cc:1398] T e85e36bb62604473be3b9a06bfd24a66 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:59:01.261845 31853 tablet_bootstrap.cc:492] T eb4abf170cb9477aad245d46676773b4 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:01.263553 31853 tablet_bootstrap.cc:654] T eb4abf170cb9477aad245d46676773b4 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:01.265400 31853 tablet_bootstrap.cc:492] T eb4abf170cb9477aad245d46676773b4 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:01.265542 31853 ts_tablet_manager.cc:1367] T eb4abf170cb9477aad245d46676773b4 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.002s
I20240502 07:59:01.265933 31853 raft_consensus.cc:348] T eb4abf170cb9477aad245d46676773b4 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:01.266037 31853 raft_consensus.cc:374] T eb4abf170cb9477aad245d46676773b4 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:01.266086 31853 raft_consensus.cc:729] T eb4abf170cb9477aad245d46676773b4 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:01.266208 31853 consensus_queue.cc:260] T eb4abf170cb9477aad245d46676773b4 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:01.266444 31853 ts_tablet_manager.cc:1398] T eb4abf170cb9477aad245d46676773b4 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:01.268718 22564 tablet_service.cc:1467] Processing CreateTablet for tablet aff9e45582034d78b16c408b3dcc4c39 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=227001975dcf4d4c859fb6fd41ff9c83]), partition=
I20240502 07:59:01.268919 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aff9e45582034d78b16c408b3dcc4c39. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:01.268993 22828 tablet_service.cc:1467] Processing CreateTablet for tablet aff9e45582034d78b16c408b3dcc4c39 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=227001975dcf4d4c859fb6fd41ff9c83]), partition=
I20240502 07:59:01.269203 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aff9e45582034d78b16c408b3dcc4c39. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:01.270346 31855 tablet_bootstrap.cc:492] T aa32346334994db4ace760fd2f08982f P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:01.270416 31855 ts_tablet_manager.cc:1367] T aa32346334994db4ace760fd2f08982f P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.011s user 0.001s sys 0.000s
I20240502 07:59:01.270867 31855 raft_consensus.cc:348] T aa32346334994db4ace760fd2f08982f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:01.270968 31855 raft_consensus.cc:374] T aa32346334994db4ace760fd2f08982f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:01.270998 31855 raft_consensus.cc:729] T aa32346334994db4ace760fd2f08982f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:01.271071 22695 tablet_service.cc:1467] Processing CreateTablet for tablet aff9e45582034d78b16c408b3dcc4c39 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=227001975dcf4d4c859fb6fd41ff9c83]), partition=
I20240502 07:59:01.271127 31854 tablet_bootstrap.cc:492] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:01.271155 31873 tablet_bootstrap.cc:492] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:01.271121 31855 consensus_queue.cc:260] T aa32346334994db4ace760fd2f08982f P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:01.271297 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aff9e45582034d78b16c408b3dcc4c39. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:01.271893 31854 tablet_bootstrap.cc:654] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:01.272308 31873 tablet_bootstrap.cc:654] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:01.273510 31855 ts_tablet_manager.cc:1398] T aa32346334994db4ace760fd2f08982f P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:59:01.274231 31876 tablet_bootstrap.cc:492] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:01.275439 31876 tablet_bootstrap.cc:654] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:01.276104 31857 tablet_bootstrap.cc:492] T eb4abf170cb9477aad245d46676773b4 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:01.276192 31857 ts_tablet_manager.cc:1367] T eb4abf170cb9477aad245d46676773b4 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.026s user 0.002s sys 0.000s
I20240502 07:59:01.276571 31857 raft_consensus.cc:348] T eb4abf170cb9477aad245d46676773b4 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:01.276683 31857 raft_consensus.cc:374] T eb4abf170cb9477aad245d46676773b4 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:01.276716 31857 raft_consensus.cc:729] T eb4abf170cb9477aad245d46676773b4 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:01.276804 31857 consensus_queue.cc:260] T eb4abf170cb9477aad245d46676773b4 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:01.277046 31857 ts_tablet_manager.cc:1398] T eb4abf170cb9477aad245d46676773b4 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:01.278712 31609 consensus_queue.cc:1035] T 9008dd5678d8495481e1b66212dc0899 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:01.279223 31873 tablet_bootstrap.cc:492] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:01.279421 31873 ts_tablet_manager.cc:1367] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.000s
I20240502 07:59:01.281374 31873 raft_consensus.cc:348] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:01.282318 31854 tablet_bootstrap.cc:492] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:01.282399 31854 ts_tablet_manager.cc:1367] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.011s user 0.001s sys 0.000s
I20240502 07:59:01.282770 31854 raft_consensus.cc:348] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:01.282894 31854 raft_consensus.cc:374] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:01.282931 31854 raft_consensus.cc:729] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:01.282876 31873 raft_consensus.cc:374] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:01.283057 31873 raft_consensus.cc:729] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:01.283049 31854 consensus_queue.cc:260] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:01.283282 31854 ts_tablet_manager.cc:1398] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:01.283257 31873 consensus_queue.cc:260] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:01.285523 31876 tablet_bootstrap.cc:492] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:01.285624 31876 ts_tablet_manager.cc:1367] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.000s
I20240502 07:59:01.285984 31876 raft_consensus.cc:348] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:01.286082 31876 raft_consensus.cc:374] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:01.286104 31876 raft_consensus.cc:729] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:01.286212 31876 consensus_queue.cc:260] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:01.286495 31876 ts_tablet_manager.cc:1398] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:01.286806 31358 raft_consensus.cc:482] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:01.286902 31358 raft_consensus.cc:504] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:01.287228 31358 leader_election.cc:288] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:01.287367 31873 ts_tablet_manager.cc:1398] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.007s user 0.001s sys 0.000s
I20240502 07:59:01.287567 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa32346334994db4ace760fd2f08982f" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:01.287667 25133 raft_consensus.cc:2454] T aa32346334994db4ace760fd2f08982f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:01.287942 24808 leader_election.cc:302] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:01.288081 31358 raft_consensus.cc:2781] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:01.288074 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa32346334994db4ace760fd2f08982f" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:01.288126 31358 raft_consensus.cc:482] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:01.288157 31358 raft_consensus.cc:3037] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:01.288174 25266 raft_consensus.cc:2454] T aa32346334994db4ace760fd2f08982f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:01.289121 31358 raft_consensus.cc:504] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:01.289402 31358 leader_election.cc:288] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:01.289845 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa32346334994db4ace760fd2f08982f" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:01.289947 25133 raft_consensus.cc:3037] T aa32346334994db4ace760fd2f08982f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:01.290830 25133 raft_consensus.cc:2454] T aa32346334994db4ace760fd2f08982f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:01.291075 24808 leader_election.cc:302] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:01.291582 31358 raft_consensus.cc:2781] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:01.291664 31358 raft_consensus.cc:686] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:59:01.291801 31358 consensus_queue.cc:237] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:01.292037 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa32346334994db4ace760fd2f08982f" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:01.292157 25266 raft_consensus.cc:3037] T aa32346334994db4ace760fd2f08982f P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:01.293135 25266 raft_consensus.cc:2454] T aa32346334994db4ace760fd2f08982f P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:01.293993 22435 catalog_manager.cc:5484] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:01.306592 31027 raft_consensus.cc:482] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:01.306672 31027 raft_consensus.cc:504] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:01.306985 31027 leader_election.cc:288] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:01.307243 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aff9e45582034d78b16c408b3dcc4c39" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:01.307294 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aff9e45582034d78b16c408b3dcc4c39" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:01.307349 22586 raft_consensus.cc:2454] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:01.307420 22716 raft_consensus.cc:2454] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:01.307580 22784 leader_election.cc:302] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:01.307765 31027 raft_consensus.cc:2781] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:01.307802 31027 raft_consensus.cc:482] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:01.307824 31027 raft_consensus.cc:3037] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:01.308476 31027 raft_consensus.cc:504] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:01.308779 31027 leader_election.cc:288] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:01.309005 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aff9e45582034d78b16c408b3dcc4c39" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:01.309026 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aff9e45582034d78b16c408b3dcc4c39" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:01.309105 22716 raft_consensus.cc:3037] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:01.309118 22586 raft_consensus.cc:3037] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:01.310045 22716 raft_consensus.cc:2454] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:01.310204 22586 raft_consensus.cc:2454] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:01.310336 22781 leader_election.cc:302] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:01.310492 31027 raft_consensus.cc:2781] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:01.310585 31027 raft_consensus.cc:686] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:59:01.310777 31027 consensus_queue.cc:237] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:01.310848 31495 raft_consensus.cc:482] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:01.310935 31495 raft_consensus.cc:504] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:01.311290 31495 leader_election.cc:288] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:01.311654 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb4abf170cb9477aad245d46676773b4" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:01.311779 25266 raft_consensus.cc:2454] T eb4abf170cb9477aad245d46676773b4 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:01.311753 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb4abf170cb9477aad245d46676773b4" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:01.311875 22978 raft_consensus.cc:2454] T eb4abf170cb9477aad245d46676773b4 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:01.312103 22521 leader_election.cc:302] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7eced220cf1642f3873fc385b6191e69, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:01.312290 31495 raft_consensus.cc:2781] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:01.312332 31495 raft_consensus.cc:482] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:01.312371 31495 raft_consensus.cc:3037] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:01.313318 31495 raft_consensus.cc:504] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:01.313519 22435 catalog_manager.cc:5484] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:01.313596 31495 leader_election.cc:288] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:01.313978 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb4abf170cb9477aad245d46676773b4" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:01.314080 22978 raft_consensus.cc:3037] T eb4abf170cb9477aad245d46676773b4 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:01.314286 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb4abf170cb9477aad245d46676773b4" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:01.314397 25266 raft_consensus.cc:3037] T eb4abf170cb9477aad245d46676773b4 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:01.314924 22978 raft_consensus.cc:2454] T eb4abf170cb9477aad245d46676773b4 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:01.315315 25266 raft_consensus.cc:2454] T eb4abf170cb9477aad245d46676773b4 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:01.315867 22520 leader_election.cc:302] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:01.316023 31495 raft_consensus.cc:2781] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:01.316092 31495 raft_consensus.cc:686] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:59:01.316223 31495 consensus_queue.cc:237] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:01.317976 22435 catalog_manager.cc:5484] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:01.422962 22435 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:59:01.426831 22586 raft_consensus.cc:1261] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:01.427146 22716 raft_consensus.cc:1261] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:01.427495 31609 consensus_queue.cc:1035] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:01.427863 31027 consensus_queue.cc:1035] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:01.430356 22444 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet aff9e45582034d78b16c408b3dcc4c39 (table rebalancer_dynamic_table [id=227001975dcf4d4c859fb6fd41ff9c83]). Expected version 1 got 0
I20240502 07:59:01.430853 22436 catalog_manager.cc:5576] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451) does not have the latest schema for tablet aff9e45582034d78b16c408b3dcc4c39 (table rebalancer_dynamic_table [id=227001975dcf4d4c859fb6fd41ff9c83]). Expected version 1 got 0
I20240502 07:59:01.432021 31891 tablet.cc:1722] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:01.432296 31891 tablet.cc:1620] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:59:01.432401 31791 tablet.cc:1722] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:01.432636 31791 tablet.cc:1620] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:59:01.435261 22436 catalog_manager.cc:5576] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451) does not have the latest schema for tablet aff9e45582034d78b16c408b3dcc4c39 (table rebalancer_dynamic_table [id=227001975dcf4d4c859fb6fd41ff9c83]). Expected version 1 got 0
I20240502 07:59:01.435755 22435 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=227001975dcf4d4c859fb6fd41ff9c83] alter complete (version 1)
I20240502 07:59:01.441735 31790 tablet.cc:1722] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:01.442013 31790 tablet.cc:1620] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:59:01.443939 22435 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet aff9e45582034d78b16c408b3dcc4c39 (table rebalancer_dynamic_table [id=227001975dcf4d4c859fb6fd41ff9c83]). Expected version 1 got 0
W20240502 07:59:01.452829 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:01.458703 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:59:01.501883 31626 raft_consensus.cc:482] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:01.501986 31626 raft_consensus.cc:504] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:01.502414 31626 leader_election.cc:288] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:01.502805 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e85e36bb62604473be3b9a06bfd24a66" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:01.502933 22586 raft_consensus.cc:2454] T e85e36bb62604473be3b9a06bfd24a66 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:01.502946 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e85e36bb62604473be3b9a06bfd24a66" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:01.503072 25266 raft_consensus.cc:2454] T e85e36bb62604473be3b9a06bfd24a66 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:01.503230 25071 leader_election.cc:302] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:01.503363 31626 raft_consensus.cc:2781] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:01.503417 31626 raft_consensus.cc:482] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:01.503451 31626 raft_consensus.cc:3037] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:01.504446 31626 raft_consensus.cc:504] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:01.504814 31626 leader_election.cc:288] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:01.505074 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e85e36bb62604473be3b9a06bfd24a66" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:01.505101 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e85e36bb62604473be3b9a06bfd24a66" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:01.505199 25266 raft_consensus.cc:3037] T e85e36bb62604473be3b9a06bfd24a66 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:01.505192 22586 raft_consensus.cc:3037] T e85e36bb62604473be3b9a06bfd24a66 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:01.506068 25266 raft_consensus.cc:2454] T e85e36bb62604473be3b9a06bfd24a66 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:01.506175 22586 raft_consensus.cc:2454] T e85e36bb62604473be3b9a06bfd24a66 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:01.506278 25070 leader_election.cc:302] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:01.506436 31626 raft_consensus.cc:2781] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:01.506635 31626 raft_consensus.cc:686] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:59:01.506775 31626 consensus_queue.cc:237] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
W20240502 07:59:01.508992 23026 tablet.cc:2369] T eb4abf170cb9477aad245d46676773b4 P 02a92e1c084b4308b9e855a4cc2c295f: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 07:59:01.509065 22435 catalog_manager.cc:5484] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:01.538870 22435 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:59:01.543454 31891 tablet.cc:1722] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:01.543773 31891 tablet.cc:1620] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:59:01.543987 31790 tablet.cc:1722] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:01.544020 31791 tablet.cc:1722] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:01.544245 31790 tablet.cc:1620] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:59:01.544315 31791 tablet.cc:1620] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:59:01.548843 22435 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=227001975dcf4d4c859fb6fd41ff9c83] alter complete (version 2)
I20240502 07:59:01.649989 22435 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:59:01.655126 31891 tablet.cc:1722] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:01.655606 31790 tablet.cc:1722] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:01.657006 31791 tablet.cc:1722] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
W20240502 07:59:01.661280 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:59:01.662199 22435 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=227001975dcf4d4c859fb6fd41ff9c83] alter complete (version 3)
I20240502 07:59:01.764034 22444 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:01.764197 22444 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:01.766238 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet aff9e45582034d78b16c408b3dcc4c39
I20240502 07:59:01.766696 22828 tablet_service.cc:1514] Processing DeleteTablet for tablet aff9e45582034d78b16c408b3dcc4c39 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:01 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:59:01.766849 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet aff9e45582034d78b16c408b3dcc4c39 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:01 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:59:01.766881 22564 tablet_service.cc:1514] Processing DeleteTablet for tablet aff9e45582034d78b16c408b3dcc4c39 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:01 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:59:01.767120 31956 tablet_replica.cc:330] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:59:01.767128 31957 tablet_replica.cc:330] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:59:01.767230 31958 tablet_replica.cc:330] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:59:01.767267 31956 raft_consensus.cc:2229] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:59:01.767270 31957 raft_consensus.cc:2229] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:01.767374 31957 raft_consensus.cc:2258] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:01.767431 31958 raft_consensus.cc:2229] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:01.767529 31956 raft_consensus.cc:2258] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:01.767577 31958 raft_consensus.cc:2258] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:01.768163 31956 ts_tablet_manager.cc:1875] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:01.768330 31957 ts_tablet_manager.cc:1875] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:01.768373 31958 ts_tablet_manager.cc:1875] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:01.770226 31957 ts_tablet_manager.cc:1888] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:01.770293 31957 log.cc:1196] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/aff9e45582034d78b16c408b3dcc4c39
I20240502 07:59:01.770496 31956 ts_tablet_manager.cc:1888] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:01.770496 31958 ts_tablet_manager.cc:1888] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:01.770573 31956 log.cc:1196] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/aff9e45582034d78b16c408b3dcc4c39
I20240502 07:59:01.770573 31958 log.cc:1196] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/aff9e45582034d78b16c408b3dcc4c39
I20240502 07:59:01.770607 31957 ts_tablet_manager.cc:1909] T aff9e45582034d78b16c408b3dcc4c39 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:59:01.770968 31956 ts_tablet_manager.cc:1909] T aff9e45582034d78b16c408b3dcc4c39 P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:59:01.771216 31958 ts_tablet_manager.cc:1909] T aff9e45582034d78b16c408b3dcc4c39 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:59:01.771189 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet aff9e45582034d78b16c408b3dcc4c39 (table rebalancer_dynamic_table_new_name [id=227001975dcf4d4c859fb6fd41ff9c83]) successfully deleted
I20240502 07:59:01.771693 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet aff9e45582034d78b16c408b3dcc4c39 (table rebalancer_dynamic_table_new_name [id=227001975dcf4d4c859fb6fd41ff9c83]) successfully deleted
I20240502 07:59:01.771838 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet aff9e45582034d78b16c408b3dcc4c39 (table rebalancer_dynamic_table_new_name [id=227001975dcf4d4c859fb6fd41ff9c83]) successfully deleted
W20240502 07:59:01.787456 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:59:01.837832 31935 consensus_queue.cc:1035] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:01.839296 31495 consensus_queue.cc:1035] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:01.842401 31935 consensus_queue.cc:1035] T aa32346334994db4ace760fd2f08982f P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:01.845742 31494 consensus_queue.cc:1035] T eb4abf170cb9477aad245d46676773b4 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:01.867692 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:59:01.872453 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 41c75f9467424a5a8031c51464fe7afc (DEFAULT_TABLE table=rebalancer_dynamic_table [id=97a64f23a69649a5965ab48b393c7631]), partition=
I20240502 07:59:01.872485 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 41c75f9467424a5a8031c51464fe7afc (DEFAULT_TABLE table=rebalancer_dynamic_table [id=97a64f23a69649a5965ab48b393c7631]), partition=
I20240502 07:59:01.872749 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 41c75f9467424a5a8031c51464fe7afc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:01.872750 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 41c75f9467424a5a8031c51464fe7afc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:01.874857 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 41c75f9467424a5a8031c51464fe7afc (DEFAULT_TABLE table=rebalancer_dynamic_table [id=97a64f23a69649a5965ab48b393c7631]), partition=
I20240502 07:59:01.875061 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 41c75f9467424a5a8031c51464fe7afc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:01.876994 31971 tablet_bootstrap.cc:492] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:01.877166 31969 tablet_bootstrap.cc:492] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
W20240502 07:59:01.877234 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:59:01.878123 31971 tablet_bootstrap.cc:654] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:01.878227 31969 tablet_bootstrap.cc:654] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:01.878808 31970 tablet_bootstrap.cc:492] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:01.879107 31971 tablet_bootstrap.cc:492] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:01.879220 31971 ts_tablet_manager.cc:1367] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:01.879235 31969 tablet_bootstrap.cc:492] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:01.879307 31969 ts_tablet_manager.cc:1367] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:01.879585 31970 tablet_bootstrap.cc:654] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:01.879714 31969 raft_consensus.cc:348] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:01.879709 31971 raft_consensus.cc:348] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:01.879812 31971 raft_consensus.cc:374] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:01.879812 31969 raft_consensus.cc:374] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:01.879851 31969 raft_consensus.cc:729] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:01.879876 31971 raft_consensus.cc:729] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:01.879992 31969 consensus_queue.cc:260] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:01.880023 31971 consensus_queue.cc:260] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:01.880280 31971 ts_tablet_manager.cc:1398] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:01.880535 31970 tablet_bootstrap.cc:492] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:01.880616 31970 ts_tablet_manager.cc:1367] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:01.881058 31970 raft_consensus.cc:348] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:01.881165 31970 raft_consensus.cc:374] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:01.881197 31970 raft_consensus.cc:729] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:01.881306 31970 consensus_queue.cc:260] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:01.881556 31970 ts_tablet_manager.cc:1398] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:01.881640 31969 ts_tablet_manager.cc:1398] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
W20240502 07:59:01.919968 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:01.920277 31935 raft_consensus.cc:482] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:01.920385 31935 raft_consensus.cc:504] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:01.920904 31935 leader_election.cc:288] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:01.921321 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "41c75f9467424a5a8031c51464fe7afc" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:01.921458 25133 raft_consensus.cc:2454] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:01.921696 24808 leader_election.cc:302] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:01.921824 31935 raft_consensus.cc:2781] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:01.921875 31935 raft_consensus.cc:482] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:01.921909 31935 raft_consensus.cc:3037] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:01.922389 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "41c75f9467424a5a8031c51464fe7afc" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:01.922523 22847 raft_consensus.cc:2454] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:01.922947 31935 raft_consensus.cc:504] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:01.923223 31935 leader_election.cc:288] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:01.923478 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "41c75f9467424a5a8031c51464fe7afc" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:59:01.923518 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "41c75f9467424a5a8031c51464fe7afc" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:01.923588 22847 raft_consensus.cc:3037] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:01.923622 25135 raft_consensus.cc:3037] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:01.924305 22847 raft_consensus.cc:2454] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:01.924516 25135 raft_consensus.cc:2454] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:01.924783 24808 leader_election.cc:302] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:01.925034 31935 raft_consensus.cc:2781] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:01.925122 31935 raft_consensus.cc:686] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:59:01.925267 31935 consensus_queue.cc:237] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:01.928186 22435 catalog_manager.cc:5484] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:02.039675 22435 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:59:02.042171 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_52"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:59:02.043824 25135 raft_consensus.cc:1261] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:02.044100 22847 raft_consensus.cc:1261] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:02.044209 31696 consensus_queue.cc:1035] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:02.044456 31935 consensus_queue.cc:1035] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:02.048477 22435 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 41c75f9467424a5a8031c51464fe7afc (table rebalancer_dynamic_table [id=97a64f23a69649a5965ab48b393c7631]). Expected version 1 got 0
I20240502 07:59:02.049299 31963 mvcc.cc:205] Tried to move back new op lower bound from 7023152095407935488 to 7023152094928228352. Current Snapshot: MvccSnapshot[applied={T|T < 7023152095407935488}]
I20240502 07:59:02.049909 32009 tablet.cc:1722] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:02.050482 32009 tablet.cc:1620] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:59:02.051513 31891 tablet.cc:1722] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:02.051817 31891 tablet.cc:1620] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:59:02.051977 22444 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet 41c75f9467424a5a8031c51464fe7afc (table rebalancer_dynamic_table [id=97a64f23a69649a5965ab48b393c7631]). Expected version 1 got 0
I20240502 07:59:02.053838 32008 tablet.cc:1722] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:02.054154 32008 tablet.cc:1620] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:59:02.054307 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 0b852d879dea4f2593fee79aa2f8ad18 (DEFAULT_TABLE table=rebalancer_extra_table_52 [id=225d0e8857eb48a5b994fcb60cf7f597]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.054603 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0b852d879dea4f2593fee79aa2f8ad18. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.055179 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 1cd7967848a146b9b830ffce9b25a688 (DEFAULT_TABLE table=rebalancer_extra_table_52 [id=225d0e8857eb48a5b994fcb60cf7f597]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.055215 22826 tablet_service.cc:1467] Processing CreateTablet for tablet 4f9f6f3ce8744c98a1885608d8930f06 (DEFAULT_TABLE table=rebalancer_extra_table_52 [id=225d0e8857eb48a5b994fcb60cf7f597]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.055227 22828 tablet_service.cc:1467] Processing CreateTablet for tablet 0b852d879dea4f2593fee79aa2f8ad18 (DEFAULT_TABLE table=rebalancer_extra_table_52 [id=225d0e8857eb48a5b994fcb60cf7f597]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.055415 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1cd7967848a146b9b830ffce9b25a688. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.055537 22828 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0b852d879dea4f2593fee79aa2f8ad18. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.055763 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f9f6f3ce8744c98a1885608d8930f06. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.057063 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 4f9f6f3ce8744c98a1885608d8930f06 (DEFAULT_TABLE table=rebalancer_extra_table_52 [id=225d0e8857eb48a5b994fcb60cf7f597]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.057317 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f9f6f3ce8744c98a1885608d8930f06. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.057696 31969 tablet_bootstrap.cc:492] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:02.058516 31969 tablet_bootstrap.cc:654] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.059283 22444 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet 41c75f9467424a5a8031c51464fe7afc (table rebalancer_dynamic_table [id=97a64f23a69649a5965ab48b393c7631]). Expected version 1 got 0
I20240502 07:59:02.059460 32012 tablet_bootstrap.cc:492] T 4f9f6f3ce8744c98a1885608d8930f06 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:02.059980 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 0b852d879dea4f2593fee79aa2f8ad18 (DEFAULT_TABLE table=rebalancer_extra_table_52 [id=225d0e8857eb48a5b994fcb60cf7f597]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.060262 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0b852d879dea4f2593fee79aa2f8ad18. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.060295 32012 tablet_bootstrap.cc:654] T 4f9f6f3ce8744c98a1885608d8930f06 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.060523 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 1cd7967848a146b9b830ffce9b25a688 (DEFAULT_TABLE table=rebalancer_extra_table_52 [id=225d0e8857eb48a5b994fcb60cf7f597]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.061838 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1cd7967848a146b9b830ffce9b25a688. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.062095 31969 tablet_bootstrap.cc:492] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:02.062167 31969 ts_tablet_manager.cc:1367] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:59:02.062450 32014 tablet_bootstrap.cc:492] T 0b852d879dea4f2593fee79aa2f8ad18 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:02.059980 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 4f9f6f3ce8744c98a1885608d8930f06 (DEFAULT_TABLE table=rebalancer_extra_table_52 [id=225d0e8857eb48a5b994fcb60cf7f597]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.062621 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f9f6f3ce8744c98a1885608d8930f06. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.062577 31969 raft_consensus.cc:348] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.062693 31969 raft_consensus.cc:374] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.062721 31969 raft_consensus.cc:729] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.062827 31969 consensus_queue.cc:260] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.063059 31969 ts_tablet_manager.cc:1398] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:02.063182 31969 tablet_bootstrap.cc:492] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:02.063292 32014 tablet_bootstrap.cc:654] T 0b852d879dea4f2593fee79aa2f8ad18 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.064129 31969 tablet_bootstrap.cc:654] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
W20240502 07:59:02.064764 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:02.065176 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 1cd7967848a146b9b830ffce9b25a688 (DEFAULT_TABLE table=rebalancer_extra_table_52 [id=225d0e8857eb48a5b994fcb60cf7f597]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.065462 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1cd7967848a146b9b830ffce9b25a688. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.065537 22435 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet 41c75f9467424a5a8031c51464fe7afc (table rebalancer_dynamic_table [id=97a64f23a69649a5965ab48b393c7631]). Expected version 1 got 0
I20240502 07:59:02.065761 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=97a64f23a69649a5965ab48b393c7631] alter complete (version 1)
I20240502 07:59:02.066802 32010 tablet_bootstrap.cc:492] T 0b852d879dea4f2593fee79aa2f8ad18 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:02.067009 32015 tablet_bootstrap.cc:492] T 1cd7967848a146b9b830ffce9b25a688 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:02.067910 32014 tablet_bootstrap.cc:492] T 0b852d879dea4f2593fee79aa2f8ad18 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:02.068074 32014 ts_tablet_manager.cc:1367] T 0b852d879dea4f2593fee79aa2f8ad18 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:59:02.068555 32014 raft_consensus.cc:348] T 0b852d879dea4f2593fee79aa2f8ad18 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.068744 32014 raft_consensus.cc:374] T 0b852d879dea4f2593fee79aa2f8ad18 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.068823 32014 raft_consensus.cc:729] T 0b852d879dea4f2593fee79aa2f8ad18 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.068981 32014 consensus_queue.cc:260] T 0b852d879dea4f2593fee79aa2f8ad18 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.070514 32015 tablet_bootstrap.cc:654] T 1cd7967848a146b9b830ffce9b25a688 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.070705 31969 tablet_bootstrap.cc:492] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:02.070784 31969 ts_tablet_manager.cc:1367] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.000s
I20240502 07:59:02.071174 31969 raft_consensus.cc:348] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.071272 31969 raft_consensus.cc:374] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.071308 31969 raft_consensus.cc:729] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.071441 31969 consensus_queue.cc:260] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.071633 32010 tablet_bootstrap.cc:654] T 0b852d879dea4f2593fee79aa2f8ad18 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.071733 31969 ts_tablet_manager.cc:1398] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:02.071843 32014 ts_tablet_manager.cc:1398] T 0b852d879dea4f2593fee79aa2f8ad18 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:59:02.071882 31969 tablet_bootstrap.cc:492] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:02.072988 32014 tablet_bootstrap.cc:492] T 4f9f6f3ce8744c98a1885608d8930f06 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:02.073005 32010 tablet_bootstrap.cc:492] T 0b852d879dea4f2593fee79aa2f8ad18 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:02.073076 32010 ts_tablet_manager.cc:1367] T 0b852d879dea4f2593fee79aa2f8ad18 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:59:02.073493 32015 tablet_bootstrap.cc:492] T 1cd7967848a146b9b830ffce9b25a688 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:02.073591 32015 ts_tablet_manager.cc:1367] T 1cd7967848a146b9b830ffce9b25a688 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
I20240502 07:59:02.073971 32014 tablet_bootstrap.cc:654] T 4f9f6f3ce8744c98a1885608d8930f06 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.074409 32010 raft_consensus.cc:348] T 0b852d879dea4f2593fee79aa2f8ad18 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.074537 32010 raft_consensus.cc:374] T 0b852d879dea4f2593fee79aa2f8ad18 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.074579 32010 raft_consensus.cc:729] T 0b852d879dea4f2593fee79aa2f8ad18 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.074898 32015 raft_consensus.cc:348] T 1cd7967848a146b9b830ffce9b25a688 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.075054 32015 raft_consensus.cc:374] T 1cd7967848a146b9b830ffce9b25a688 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.075137 32015 raft_consensus.cc:729] T 1cd7967848a146b9b830ffce9b25a688 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.075286 32014 tablet_bootstrap.cc:492] T 4f9f6f3ce8744c98a1885608d8930f06 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:02.075302 32015 consensus_queue.cc:260] T 1cd7967848a146b9b830ffce9b25a688 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.075420 32014 ts_tablet_manager.cc:1367] T 4f9f6f3ce8744c98a1885608d8930f06 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:02.075877 31969 tablet_bootstrap.cc:654] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.075893 32014 raft_consensus.cc:348] T 4f9f6f3ce8744c98a1885608d8930f06 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.075984 32014 raft_consensus.cc:374] T 4f9f6f3ce8744c98a1885608d8930f06 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.076046 32014 raft_consensus.cc:729] T 4f9f6f3ce8744c98a1885608d8930f06 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.076208 32014 consensus_queue.cc:260] T 4f9f6f3ce8744c98a1885608d8930f06 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.076548 32015 ts_tablet_manager.cc:1398] T 1cd7967848a146b9b830ffce9b25a688 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:59:02.074692 32010 consensus_queue.cc:260] T 0b852d879dea4f2593fee79aa2f8ad18 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.076972 32010 ts_tablet_manager.cc:1398] T 0b852d879dea4f2593fee79aa2f8ad18 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.003s user 0.000s sys 0.001s
I20240502 07:59:02.077167 32010 tablet_bootstrap.cc:492] T 1cd7967848a146b9b830ffce9b25a688 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:02.077231 32014 ts_tablet_manager.cc:1398] T 4f9f6f3ce8744c98a1885608d8930f06 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:02.078862 32010 tablet_bootstrap.cc:654] T 1cd7967848a146b9b830ffce9b25a688 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.082160 31609 raft_consensus.cc:482] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:02.082234 32012 tablet_bootstrap.cc:492] T 4f9f6f3ce8744c98a1885608d8930f06 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:02.082239 31609 raft_consensus.cc:504] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.082322 32012 ts_tablet_manager.cc:1367] T 4f9f6f3ce8744c98a1885608d8930f06 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.023s user 0.001s sys 0.000s
I20240502 07:59:02.082641 31609 leader_election.cc:288] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:59:02.082700 31969 tablet_bootstrap.cc:492] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:02.082768 31969 ts_tablet_manager.cc:1367] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.011s user 0.001s sys 0.000s
I20240502 07:59:02.082759 32012 raft_consensus.cc:348] T 4f9f6f3ce8744c98a1885608d8930f06 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.082860 32012 raft_consensus.cc:374] T 4f9f6f3ce8744c98a1885608d8930f06 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.082886 32012 raft_consensus.cc:729] T 4f9f6f3ce8744c98a1885608d8930f06 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.082984 32012 consensus_queue.cc:260] T 4f9f6f3ce8744c98a1885608d8930f06 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.083140 31969 raft_consensus.cc:348] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.083173 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f9f6f3ce8744c98a1885608d8930f06" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:02.083207 32012 ts_tablet_manager.cc:1398] T 4f9f6f3ce8744c98a1885608d8930f06 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:02.083223 31969 raft_consensus.cc:374] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.083254 31969 raft_consensus.cc:729] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.083251 22586 raft_consensus.cc:2454] T 4f9f6f3ce8744c98a1885608d8930f06 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:02.083352 31969 consensus_queue.cc:260] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.083505 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f9f6f3ce8744c98a1885608d8930f06" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:02.083604 22716 raft_consensus.cc:2454] T 4f9f6f3ce8744c98a1885608d8930f06 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:02.083617 31969 ts_tablet_manager.cc:1398] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:02.084170 32010 tablet_bootstrap.cc:492] T 1cd7967848a146b9b830ffce9b25a688 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:02.084247 32010 ts_tablet_manager.cc:1367] T 1cd7967848a146b9b830ffce9b25a688 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.001s
I20240502 07:59:02.084472 22784 leader_election.cc:302] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:02.084725 32010 raft_consensus.cc:348] T 1cd7967848a146b9b830ffce9b25a688 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.084825 32010 raft_consensus.cc:374] T 1cd7967848a146b9b830ffce9b25a688 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.084864 32010 raft_consensus.cc:729] T 1cd7967848a146b9b830ffce9b25a688 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.084990 32010 consensus_queue.cc:260] T 1cd7967848a146b9b830ffce9b25a688 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.085191 31609 raft_consensus.cc:2781] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:02.085244 31609 raft_consensus.cc:482] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:02.085283 31609 raft_consensus.cc:3037] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.085281 32010 ts_tablet_manager.cc:1398] T 1cd7967848a146b9b830ffce9b25a688 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:02.085888 32023 consensus_queue.cc:1035] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20240502 07:59:02.086251 31609 raft_consensus.cc:504] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.086524 31609 leader_election.cc:288] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:59:02.086807 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f9f6f3ce8744c98a1885608d8930f06" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:02.086903 22586 raft_consensus.cc:3037] T 4f9f6f3ce8744c98a1885608d8930f06 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.086992 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f9f6f3ce8744c98a1885608d8930f06" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:02.087098 22716 raft_consensus.cc:3037] T 4f9f6f3ce8744c98a1885608d8930f06 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.087656 22586 raft_consensus.cc:2454] T 4f9f6f3ce8744c98a1885608d8930f06 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:02.087888 22784 leader_election.cc:302] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:02.087965 22716 raft_consensus.cc:2454] T 4f9f6f3ce8744c98a1885608d8930f06 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:02.088812 31886 raft_consensus.cc:2781] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:02.088899 31886 raft_consensus.cc:686] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:59:02.089040 31886 consensus_queue.cc:237] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.091131 22444 catalog_manager.cc:5484] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } }
I20240502 07:59:02.094972 32023 consensus_queue.cc:1035] T e85e36bb62604473be3b9a06bfd24a66 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:02.107932 31886 raft_consensus.cc:482] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:02.108039 31886 raft_consensus.cc:504] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.108457 31886 leader_election.cc:288] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:02.108757 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cd7967848a146b9b830ffce9b25a688" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:02.108789 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cd7967848a146b9b830ffce9b25a688" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:02.108891 25266 raft_consensus.cc:2454] T 1cd7967848a146b9b830ffce9b25a688 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:02.108922 22978 raft_consensus.cc:2454] T 1cd7967848a146b9b830ffce9b25a688 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:02.109133 22783 leader_election.cc:302] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:02.109321 31886 raft_consensus.cc:2781] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:02.109373 31886 raft_consensus.cc:482] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:02.109405 31886 raft_consensus.cc:3037] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.110250 31886 raft_consensus.cc:504] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.110586 31886 leader_election.cc:288] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:02.110927 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cd7967848a146b9b830ffce9b25a688" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:02.110988 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cd7967848a146b9b830ffce9b25a688" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:02.111040 22978 raft_consensus.cc:3037] T 1cd7967848a146b9b830ffce9b25a688 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.111091 25266 raft_consensus.cc:3037] T 1cd7967848a146b9b830ffce9b25a688 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.111846 22978 raft_consensus.cc:2454] T 1cd7967848a146b9b830ffce9b25a688 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:02.111974 25266 raft_consensus.cc:2454] T 1cd7967848a146b9b830ffce9b25a688 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:02.112116 22782 leader_election.cc:302] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:02.112272 31886 raft_consensus.cc:2781] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:02.112340 31886 raft_consensus.cc:686] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
W20240502 07:59:02.112346 22895 tablet.cc:2369] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 07:59:02.112462 31886 consensus_queue.cc:237] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.114640 22436 catalog_manager.cc:5484] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:02.171142 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:59:02.175814 32008 tablet.cc:1722] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:02.176158 32008 tablet.cc:1620] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:59:02.176157 32009 tablet.cc:1722] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:02.176467 32009 tablet.cc:1620] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:59:02.176692 31891 tablet.cc:1722] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:02.176964 31891 tablet.cc:1620] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:59:02.182518 22438 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=97a64f23a69649a5965ab48b393c7631] alter complete (version 2)
W20240502 07:59:02.194286 22764 tablet.cc:2369] T 0b852d879dea4f2593fee79aa2f8ad18 P 725969d490a04fdfb3ba890c3f5096e3: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 07:59:02.217248 31609 raft_consensus.cc:482] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:02.217342 31609 raft_consensus.cc:504] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.217646 31609 leader_election.cc:288] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:02.217897 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0b852d879dea4f2593fee79aa2f8ad18" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:02.217940 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0b852d879dea4f2593fee79aa2f8ad18" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:02.218009 22716 raft_consensus.cc:2454] T 0b852d879dea4f2593fee79aa2f8ad18 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:02.218060 22978 raft_consensus.cc:2454] T 0b852d879dea4f2593fee79aa2f8ad18 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:02.218225 22781 leader_election.cc:302] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:02.218400 31609 raft_consensus.cc:2781] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:02.218449 31609 raft_consensus.cc:482] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:02.218478 31609 raft_consensus.cc:3037] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.219451 31609 raft_consensus.cc:504] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.219717 31609 leader_election.cc:288] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:02.220053 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0b852d879dea4f2593fee79aa2f8ad18" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:02.220165 22716 raft_consensus.cc:3037] T 0b852d879dea4f2593fee79aa2f8ad18 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.220233 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0b852d879dea4f2593fee79aa2f8ad18" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:02.220324 22978 raft_consensus.cc:3037] T 0b852d879dea4f2593fee79aa2f8ad18 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.221160 22716 raft_consensus.cc:2454] T 0b852d879dea4f2593fee79aa2f8ad18 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:02.221242 22978 raft_consensus.cc:2454] T 0b852d879dea4f2593fee79aa2f8ad18 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:02.221468 22781 leader_election.cc:302] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:02.221665 31886 raft_consensus.cc:2781] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:02.221752 31886 raft_consensus.cc:686] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:59:02.221858 31886 consensus_queue.cc:237] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.225634 22444 catalog_manager.cc:5484] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
W20240502 07:59:02.237272 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:02.286055 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
W20240502 07:59:02.286644 22633 tablet.cc:2369] T 4f9f6f3ce8744c98a1885608d8930f06 P 8bea0fe8d8044a2685f92f7a10c9cf85: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 07:59:02.290364 32008 tablet.cc:1722] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:02.290959 32009 tablet.cc:1722] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:02.291267 31891 tablet.cc:1722] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:02.296298 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=97a64f23a69649a5965ab48b393c7631] alter complete (version 3)
W20240502 07:59:02.315256 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 07:59:02.356206 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:02.398874 22436 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:02.399008 22436 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:02.401149 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 41c75f9467424a5a8031c51464fe7afc
I20240502 07:59:02.401710 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet 41c75f9467424a5a8031c51464fe7afc with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:02 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:59:02.401746 25115 tablet_service.cc:1514] Processing DeleteTablet for tablet 41c75f9467424a5a8031c51464fe7afc with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:02 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:59:02.402033 32063 tablet_replica.cc:330] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:59:02.402006 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet 41c75f9467424a5a8031c51464fe7afc with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:02 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:59:02.402197 32063 raft_consensus.cc:2229] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:02.402312 32065 tablet_replica.cc:330] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:59:02.402318 32063 raft_consensus.cc:2258] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:02.402467 32065 raft_consensus.cc:2229] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:59:02.402547 32064 tablet_replica.cc:330] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:59:02.402724 32065 raft_consensus.cc:2258] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:02.402748 32064 raft_consensus.cc:2229] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:02.402858 32064 raft_consensus.cc:2258] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:02.402880 32063 ts_tablet_manager.cc:1875] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:02.403196 32065 ts_tablet_manager.cc:1875] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:02.403460 32064 ts_tablet_manager.cc:1875] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:02.405339 32065 ts_tablet_manager.cc:1888] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:02.405404 32065 log.cc:1196] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/41c75f9467424a5a8031c51464fe7afc
I20240502 07:59:02.405488 32063 ts_tablet_manager.cc:1888] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:02.405587 32063 log.cc:1196] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/41c75f9467424a5a8031c51464fe7afc
I20240502 07:59:02.405702 32064 ts_tablet_manager.cc:1888] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:02.405753 32065 ts_tablet_manager.cc:1909] T 41c75f9467424a5a8031c51464fe7afc P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:59:02.405772 32064 log.cc:1196] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/41c75f9467424a5a8031c51464fe7afc
I20240502 07:59:02.405970 32063 ts_tablet_manager.cc:1909] T 41c75f9467424a5a8031c51464fe7afc P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:59:02.405990 32064 ts_tablet_manager.cc:1909] T 41c75f9467424a5a8031c51464fe7afc P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:59:02.406308 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet 41c75f9467424a5a8031c51464fe7afc (table rebalancer_dynamic_table_new_name [id=97a64f23a69649a5965ab48b393c7631]) successfully deleted
I20240502 07:59:02.406570 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet 41c75f9467424a5a8031c51464fe7afc (table rebalancer_dynamic_table_new_name [id=97a64f23a69649a5965ab48b393c7631]) successfully deleted
I20240502 07:59:02.406716 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 41c75f9467424a5a8031c51464fe7afc (table rebalancer_dynamic_table_new_name [id=97a64f23a69649a5965ab48b393c7631]) successfully deleted
W20240502 07:59:02.417459 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 07:59:02.425397 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 07:59:02.454978 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 07:59:02.464439 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:02.494921 31886 consensus_queue.cc:1035] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:02.497840 31886 consensus_queue.cc:1035] T 1cd7967848a146b9b830ffce9b25a688 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:02.502774 22438 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:59:02.507352 22697 tablet_service.cc:1467] Processing CreateTablet for tablet d0afadaff02a4abbaf34a35f647a8c6c (DEFAULT_TABLE table=rebalancer_dynamic_table [id=1600c880fda4465789f2d598c33d387f]), partition=
I20240502 07:59:02.507455 22564 tablet_service.cc:1467] Processing CreateTablet for tablet d0afadaff02a4abbaf34a35f647a8c6c (DEFAULT_TABLE table=rebalancer_dynamic_table [id=1600c880fda4465789f2d598c33d387f]), partition=
I20240502 07:59:02.507598 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d0afadaff02a4abbaf34a35f647a8c6c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.507694 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d0afadaff02a4abbaf34a35f647a8c6c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.507685 22959 tablet_service.cc:1467] Processing CreateTablet for tablet d0afadaff02a4abbaf34a35f647a8c6c (DEFAULT_TABLE table=rebalancer_dynamic_table [id=1600c880fda4465789f2d598c33d387f]), partition=
I20240502 07:59:02.507855 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d0afadaff02a4abbaf34a35f647a8c6c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.509675 32014 tablet_bootstrap.cc:492] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:02.509756 32012 tablet_bootstrap.cc:492] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:02.509821 32010 tablet_bootstrap.cc:492] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:02.510330 32014 tablet_bootstrap.cc:654] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.510507 32012 tablet_bootstrap.cc:654] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.510507 32010 tablet_bootstrap.cc:654] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.511739 32012 tablet_bootstrap.cc:492] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:02.511814 32012 ts_tablet_manager.cc:1367] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:02.512276 32010 tablet_bootstrap.cc:492] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:02.512243 32012 raft_consensus.cc:348] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.512328 32012 raft_consensus.cc:374] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.512334 32010 ts_tablet_manager.cc:1367] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.001s
I20240502 07:59:02.512358 32012 raft_consensus.cc:729] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.512471 32012 consensus_queue.cc:260] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.512742 32012 ts_tablet_manager.cc:1398] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:02.512740 32010 raft_consensus.cc:348] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.512847 32010 raft_consensus.cc:374] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.512884 32010 raft_consensus.cc:729] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.512985 32010 consensus_queue.cc:260] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.513196 32010 ts_tablet_manager.cc:1398] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:02.514240 32014 tablet_bootstrap.cc:492] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:02.514310 32014 ts_tablet_manager.cc:1367] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:59:02.514760 32014 raft_consensus.cc:348] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.514861 32014 raft_consensus.cc:374] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.514896 32014 raft_consensus.cc:729] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.514998 32014 consensus_queue.cc:260] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.515264 32014 ts_tablet_manager.cc:1398] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
W20240502 07:59:02.523257 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 07:59:02.557160 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:02.569980 30685 raft_consensus.cc:482] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:02.570075 30685 raft_consensus.cc:504] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.570475 30685 leader_election.cc:288] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:59:02.570739 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d0afadaff02a4abbaf34a35f647a8c6c" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:02.570776 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d0afadaff02a4abbaf34a35f647a8c6c" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:02.570852 22716 raft_consensus.cc:2454] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:59:02.570892 22584 raft_consensus.cc:2454] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:59:02.571112 22912 leader_election.cc:302] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:02.571272 30685 raft_consensus.cc:2781] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:02.571331 30685 raft_consensus.cc:482] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:02.571364 30685 raft_consensus.cc:3037] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.572297 30685 raft_consensus.cc:504] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.572599 30685 leader_election.cc:288] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:59:02.572986 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d0afadaff02a4abbaf34a35f647a8c6c" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:02.573081 22716 raft_consensus.cc:3037] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.573138 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d0afadaff02a4abbaf34a35f647a8c6c" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:02.573232 22584 raft_consensus.cc:3037] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.574007 22716 raft_consensus.cc:2454] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:59:02.574039 22584 raft_consensus.cc:2454] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:59:02.574258 22915 leader_election.cc:302] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:02.574506 30685 raft_consensus.cc:2781] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:02.574646 30685 raft_consensus.cc:686] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:59:02.574767 30685 consensus_queue.cc:237] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:02.576944 22436 catalog_manager.cc:5484] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } }
W20240502 07:59:02.593875 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:02.607558 31609 consensus_queue.cc:1035] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:02.610632 31609 consensus_queue.cc:1035] T 4f9f6f3ce8744c98a1885608d8930f06 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:02.689265 22438 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:59:02.694664 22716 raft_consensus.cc:1261] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:02.694916 22584 raft_consensus.cc:1261] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:02.695083 32102 consensus_queue.cc:1035] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:02.695367 32102 consensus_queue.cc:1035] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:02.698369 22438 catalog_manager.cc:5576] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451) does not have the latest schema for tablet d0afadaff02a4abbaf34a35f647a8c6c (table rebalancer_dynamic_table [id=1600c880fda4465789f2d598c33d387f]). Expected version 1 got 0
I20240502 07:59:02.698844 22436 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet d0afadaff02a4abbaf34a35f647a8c6c (table rebalancer_dynamic_table [id=1600c880fda4465789f2d598c33d387f]). Expected version 1 got 0
I20240502 07:59:02.699844 32130 tablet.cc:1722] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:02.700148 32130 tablet.cc:1620] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:02.700107 32131 tablet.cc:1722] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:02.700382 32131 tablet.cc:1620] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:59:02.702617 22438 catalog_manager.cc:5576] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451) does not have the latest schema for tablet d0afadaff02a4abbaf34a35f647a8c6c (table rebalancer_dynamic_table [id=1600c880fda4465789f2d598c33d387f]). Expected version 1 got 0
I20240502 07:59:02.705186 22436 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet d0afadaff02a4abbaf34a35f647a8c6c (table rebalancer_dynamic_table [id=1600c880fda4465789f2d598c33d387f]). Expected version 1 got 0
I20240502 07:59:02.705596 22435 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet d0afadaff02a4abbaf34a35f647a8c6c (table rebalancer_dynamic_table [id=1600c880fda4465789f2d598c33d387f]). Expected version 1 got 0
I20240502 07:59:02.706589 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=1600c880fda4465789f2d598c33d387f] alter complete (version 1)
I20240502 07:59:02.707494 32132 tablet.cc:1722] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:02.707815 32132 tablet.cc:1620] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:59:02.765184 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_54"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:59:02.770812 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 69fad10dadc74d24a783fd05bc36fa9a (DEFAULT_TABLE table=rebalancer_extra_table_54 [id=6039d0994c664479822bc35153bbb891]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.770929 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 69fad10dadc74d24a783fd05bc36fa9a (DEFAULT_TABLE table=rebalancer_extra_table_54 [id=6039d0994c664479822bc35153bbb891]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.771100 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 69fad10dadc74d24a783fd05bc36fa9a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.771132 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 69fad10dadc74d24a783fd05bc36fa9a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.771852 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 69fad10dadc74d24a783fd05bc36fa9a (DEFAULT_TABLE table=rebalancer_extra_table_54 [id=6039d0994c664479822bc35153bbb891]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.772100 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 69fad10dadc74d24a783fd05bc36fa9a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.773216 25235 tablet_service.cc:1467] Processing CreateTablet for tablet d2dfb8781b2f4f7ab1da2c8fb68193fa (DEFAULT_TABLE table=rebalancer_extra_table_54 [id=6039d0994c664479822bc35153bbb891]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.773206 22697 tablet_service.cc:1467] Processing CreateTablet for tablet d2dfb8781b2f4f7ab1da2c8fb68193fa (DEFAULT_TABLE table=rebalancer_extra_table_54 [id=6039d0994c664479822bc35153bbb891]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.773320 22564 tablet_service.cc:1467] Processing CreateTablet for tablet d2dfb8781b2f4f7ab1da2c8fb68193fa (DEFAULT_TABLE table=rebalancer_extra_table_54 [id=6039d0994c664479822bc35153bbb891]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.773413 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d2dfb8781b2f4f7ab1da2c8fb68193fa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.773476 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d2dfb8781b2f4f7ab1da2c8fb68193fa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.773547 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d2dfb8781b2f4f7ab1da2c8fb68193fa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.775049 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 28dcd73d0afc4d36a659bf5fc2daa897 (DEFAULT_TABLE table=rebalancer_extra_table_54 [id=6039d0994c664479822bc35153bbb891]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.775115 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 28dcd73d0afc4d36a659bf5fc2daa897 (DEFAULT_TABLE table=rebalancer_extra_table_54 [id=6039d0994c664479822bc35153bbb891]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.775285 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 28dcd73d0afc4d36a659bf5fc2daa897. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.775367 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 28dcd73d0afc4d36a659bf5fc2daa897. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.777061 32012 tablet_bootstrap.cc:492] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:02.777941 32145 tablet_bootstrap.cc:492] T 28dcd73d0afc4d36a659bf5fc2daa897 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:02.778023 32012 tablet_bootstrap.cc:654] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.778117 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 28dcd73d0afc4d36a659bf5fc2daa897 (DEFAULT_TABLE table=rebalancer_extra_table_54 [id=6039d0994c664479822bc35153bbb891]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:02.778327 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 28dcd73d0afc4d36a659bf5fc2daa897. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:02.779323 32014 tablet_bootstrap.cc:492] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:02.779834 32145 tablet_bootstrap.cc:654] T 28dcd73d0afc4d36a659bf5fc2daa897 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.780207 32014 tablet_bootstrap.cc:654] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.780225 32146 tablet_bootstrap.cc:492] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:02.781075 32145 tablet_bootstrap.cc:492] T 28dcd73d0afc4d36a659bf5fc2daa897 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:02.781150 32145 ts_tablet_manager.cc:1367] T 28dcd73d0afc4d36a659bf5fc2daa897 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:59:02.781170 32146 tablet_bootstrap.cc:654] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.781587 32145 raft_consensus.cc:348] T 28dcd73d0afc4d36a659bf5fc2daa897 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.781687 32145 raft_consensus.cc:374] T 28dcd73d0afc4d36a659bf5fc2daa897 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.781718 32145 raft_consensus.cc:729] T 28dcd73d0afc4d36a659bf5fc2daa897 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.781831 32145 consensus_queue.cc:260] T 28dcd73d0afc4d36a659bf5fc2daa897 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.782030 32143 tablet_bootstrap.cc:492] T 69fad10dadc74d24a783fd05bc36fa9a P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:02.782095 32145 ts_tablet_manager.cc:1398] T 28dcd73d0afc4d36a659bf5fc2daa897 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:02.782977 32143 tablet_bootstrap.cc:654] T 69fad10dadc74d24a783fd05bc36fa9a P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.783176 32014 tablet_bootstrap.cc:492] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:02.783252 32014 ts_tablet_manager.cc:1367] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.001s
I20240502 07:59:02.783653 32012 tablet_bootstrap.cc:492] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:02.783728 32012 ts_tablet_manager.cc:1367] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
I20240502 07:59:02.783706 32014 raft_consensus.cc:348] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:02.783803 32014 raft_consensus.cc:374] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.783851 32014 raft_consensus.cc:729] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.783962 32014 consensus_queue.cc:260] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:02.784197 32014 ts_tablet_manager.cc:1398] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:02.784183 32012 raft_consensus.cc:348] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:02.784277 32012 raft_consensus.cc:374] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.784312 32012 raft_consensus.cc:729] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.784427 32012 consensus_queue.cc:260] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:02.784694 32144 tablet_bootstrap.cc:492] T 69fad10dadc74d24a783fd05bc36fa9a P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:02.784839 32012 ts_tablet_manager.cc:1398] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:02.785756 32146 tablet_bootstrap.cc:492] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:02.785883 32146 ts_tablet_manager.cc:1367] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.002s
I20240502 07:59:02.786335 32146 raft_consensus.cc:348] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:02.786566 32146 raft_consensus.cc:374] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.786636 32146 raft_consensus.cc:729] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.786788 32146 consensus_queue.cc:260] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:02.787590 32144 tablet_bootstrap.cc:654] T 69fad10dadc74d24a783fd05bc36fa9a P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.788733 32144 tablet_bootstrap.cc:492] T 69fad10dadc74d24a783fd05bc36fa9a P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:02.788807 32144 ts_tablet_manager.cc:1367] T 69fad10dadc74d24a783fd05bc36fa9a P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:59:02.788806 32146 ts_tablet_manager.cc:1398] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:59:02.788949 32146 tablet_bootstrap.cc:492] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:02.789184 32144 raft_consensus.cc:348] T 69fad10dadc74d24a783fd05bc36fa9a P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:02.789273 32144 raft_consensus.cc:374] T 69fad10dadc74d24a783fd05bc36fa9a P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.789304 32144 raft_consensus.cc:729] T 69fad10dadc74d24a783fd05bc36fa9a P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.789417 32144 consensus_queue.cc:260] T 69fad10dadc74d24a783fd05bc36fa9a P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:02.789683 32144 ts_tablet_manager.cc:1398] T 69fad10dadc74d24a783fd05bc36fa9a P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:02.789909 32146 tablet_bootstrap.cc:654] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.789947 32144 tablet_bootstrap.cc:492] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:02.790805 32144 tablet_bootstrap.cc:654] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.790990 32143 tablet_bootstrap.cc:492] T 69fad10dadc74d24a783fd05bc36fa9a P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:02.791105 32143 ts_tablet_manager.cc:1367] T 69fad10dadc74d24a783fd05bc36fa9a P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.009s user 0.002s sys 0.000s
I20240502 07:59:02.791599 32143 raft_consensus.cc:348] T 69fad10dadc74d24a783fd05bc36fa9a P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:02.791754 32143 raft_consensus.cc:374] T 69fad10dadc74d24a783fd05bc36fa9a P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.791832 32143 raft_consensus.cc:729] T 69fad10dadc74d24a783fd05bc36fa9a P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.792013 32144 tablet_bootstrap.cc:492] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:02.792085 32144 ts_tablet_manager.cc:1367] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:02.792006 32143 consensus_queue.cc:260] T 69fad10dadc74d24a783fd05bc36fa9a P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:02.792389 32143 ts_tablet_manager.cc:1398] T 69fad10dadc74d24a783fd05bc36fa9a P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:02.792486 32144 raft_consensus.cc:348] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.792579 32144 raft_consensus.cc:374] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.792613 32144 raft_consensus.cc:729] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.792723 32143 tablet_bootstrap.cc:492] T 28dcd73d0afc4d36a659bf5fc2daa897 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:02.792753 32144 consensus_queue.cc:260] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.792973 32144 ts_tablet_manager.cc:1398] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:02.793777 32143 tablet_bootstrap.cc:654] T 28dcd73d0afc4d36a659bf5fc2daa897 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:02.794497 32146 tablet_bootstrap.cc:492] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:02.794624 32146 ts_tablet_manager.cc:1367] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:59:02.795140 32146 raft_consensus.cc:348] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:02.795292 32146 raft_consensus.cc:374] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.795372 32146 raft_consensus.cc:729] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.795516 32146 consensus_queue.cc:260] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:02.795794 32146 ts_tablet_manager.cc:1398] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:02.796139 32143 tablet_bootstrap.cc:492] T 28dcd73d0afc4d36a659bf5fc2daa897 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:02.796209 32143 ts_tablet_manager.cc:1367] T 28dcd73d0afc4d36a659bf5fc2daa897 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:59:02.797291 32143 raft_consensus.cc:348] T 28dcd73d0afc4d36a659bf5fc2daa897 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.797367 32143 raft_consensus.cc:374] T 28dcd73d0afc4d36a659bf5fc2daa897 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:02.797397 32143 raft_consensus.cc:729] T 28dcd73d0afc4d36a659bf5fc2daa897 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:02.797515 32143 consensus_queue.cc:260] T 28dcd73d0afc4d36a659bf5fc2daa897 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.797776 32143 ts_tablet_manager.cc:1398] T 28dcd73d0afc4d36a659bf5fc2daa897 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:02.812517 31609 consensus_queue.cc:1035] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:02.815375 31886 consensus_queue.cc:1035] T 0b852d879dea4f2593fee79aa2f8ad18 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:02.817262 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:02.817202 22438 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:59:02.822116 32130 tablet.cc:1722] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:02.822434 32130 tablet.cc:1620] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:02.822742 32132 tablet.cc:1722] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:02.823016 32132 tablet.cc:1620] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:59:02.825817 32131 tablet.cc:1722] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:02.826105 32131 tablet.cc:1620] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:59:02.828097 22438 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=1600c880fda4465789f2d598c33d387f] alter complete (version 2)
W20240502 07:59:02.845916 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:59:02.892875 31698 raft_consensus.cc:482] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:02.892990 31698 raft_consensus.cc:504] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:02.893481 31698 leader_election.cc:288] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:02.893723 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "69fad10dadc74d24a783fd05bc36fa9a" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:02.893844 24872 raft_consensus.cc:2454] T 69fad10dadc74d24a783fd05bc36fa9a P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:59:02.894097 25199 leader_election.cc:302] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:02.894380 31698 raft_consensus.cc:2781] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:02.894462 31698 raft_consensus.cc:482] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:02.894505 31698 raft_consensus.cc:3037] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.895351 31698 raft_consensus.cc:504] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:02.895691 31698 leader_election.cc:288] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:02.895987 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "69fad10dadc74d24a783fd05bc36fa9a" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:02.896090 24872 raft_consensus.cc:3037] T 69fad10dadc74d24a783fd05bc36fa9a P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.896267 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "69fad10dadc74d24a783fd05bc36fa9a" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:02.896392 25133 raft_consensus.cc:3037] T 69fad10dadc74d24a783fd05bc36fa9a P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.896287 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "69fad10dadc74d24a783fd05bc36fa9a" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:02.897032 24872 raft_consensus.cc:2454] T 69fad10dadc74d24a783fd05bc36fa9a P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:59:02.897302 25133 raft_consensus.cc:2454] T 69fad10dadc74d24a783fd05bc36fa9a P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:59:02.897302 25199 leader_election.cc:302] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:02.897467 31793 raft_consensus.cc:2781] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:02.897554 31793 raft_consensus.cc:686] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:59:02.897696 31793 consensus_queue.cc:237] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:02.899780 22436 catalog_manager.cc:5484] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:02.928305 32026 raft_consensus.cc:482] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:02.928401 32026 raft_consensus.cc:504] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.928866 32026 leader_election.cc:288] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:02.929177 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28dcd73d0afc4d36a659bf5fc2daa897" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:02.929281 22847 raft_consensus.cc:2454] T 28dcd73d0afc4d36a659bf5fc2daa897 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:02.929481 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28dcd73d0afc4d36a659bf5fc2daa897" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:02.929587 24870 raft_consensus.cc:2454] T 28dcd73d0afc4d36a659bf5fc2daa897 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:02.929795 25068 leader_election.cc:302] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:02.930424 32026 raft_consensus.cc:2781] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:02.930482 32026 raft_consensus.cc:482] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:02.930536 32026 raft_consensus.cc:3037] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.931154 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:59:02.931565 32026 raft_consensus.cc:504] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.931880 32026 leader_election.cc:288] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:02.932181 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28dcd73d0afc4d36a659bf5fc2daa897" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:02.932286 24870 raft_consensus.cc:3037] T 28dcd73d0afc4d36a659bf5fc2daa897 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.932799 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28dcd73d0afc4d36a659bf5fc2daa897" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:59:02.932905 22847 raft_consensus.cc:3037] T 28dcd73d0afc4d36a659bf5fc2daa897 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.933446 24870 raft_consensus.cc:2454] T 28dcd73d0afc4d36a659bf5fc2daa897 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:02.933794 22847 raft_consensus.cc:2454] T 28dcd73d0afc4d36a659bf5fc2daa897 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
W20240502 07:59:02.934140 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:02.934208 25071 leader_election.cc:302] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:02.934334 32026 raft_consensus.cc:2781] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:02.934407 32026 raft_consensus.cc:686] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:59:02.934576 32026 consensus_queue.cc:237] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:02.936883 22436 catalog_manager.cc:5484] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:02.939421 32130 tablet.cc:1722] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:02.940052 32132 tablet.cc:1722] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:02.941540 32131 tablet.cc:1722] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:02.946255 22438 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=1600c880fda4465789f2d598c33d387f] alter complete (version 3)
W20240502 07:59:02.950582 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:02.959139 31793 raft_consensus.cc:482] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:02.959230 31793 raft_consensus.cc:504] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:02.959623 31793 leader_election.cc:288] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:02.959878 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d2dfb8781b2f4f7ab1da2c8fb68193fa" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:02.959901 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d2dfb8781b2f4f7ab1da2c8fb68193fa" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:02.959997 22586 raft_consensus.cc:2454] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:59:02.959997 22716 raft_consensus.cc:2454] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:59:02.960244 25199 leader_election.cc:302] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:02.960414 31793 raft_consensus.cc:2781] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:02.960482 31793 raft_consensus.cc:482] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:02.960537 31793 raft_consensus.cc:3037] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.961570 31793 raft_consensus.cc:504] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:02.961910 31793 leader_election.cc:288] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:02.962256 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d2dfb8781b2f4f7ab1da2c8fb68193fa" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:02.962249 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d2dfb8781b2f4f7ab1da2c8fb68193fa" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:02.962352 22586 raft_consensus.cc:3037] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.962356 22716 raft_consensus.cc:3037] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:02.963107 22716 raft_consensus.cc:2454] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:59:02.963138 22586 raft_consensus.cc:2454] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:59:02.963328 25199 leader_election.cc:302] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:02.963729 31698 raft_consensus.cc:2781] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:02.963793 31698 raft_consensus.cc:686] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:59:02.963896 31698 consensus_queue.cc:237] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:02.965975 22436 catalog_manager.cc:5484] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } }
W20240502 07:59:02.998541 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 07:59:03.004400 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:03.048180 22436 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:03.048326 22436 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:03.050139 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet d0afadaff02a4abbaf34a35f647a8c6c
I20240502 07:59:03.050653 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet d0afadaff02a4abbaf34a35f647a8c6c with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:03 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:59:03.050689 22564 tablet_service.cc:1514] Processing DeleteTablet for tablet d0afadaff02a4abbaf34a35f647a8c6c with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:03 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:59:03.050787 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet d0afadaff02a4abbaf34a35f647a8c6c with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:03 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:59:03.050978 32196 tablet_replica.cc:330] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:59:03.051038 32195 tablet_replica.cc:330] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:59:03.051070 32197 tablet_replica.cc:330] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:59:03.051129 32196 raft_consensus.cc:2229] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:03.051192 32195 raft_consensus.cc:2229] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:03.051191 32197 raft_consensus.cc:2229] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:59:03.051249 32196 raft_consensus.cc:2258] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:03.051301 32195 raft_consensus.cc:2258] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:03.051617 32197 raft_consensus.cc:2258] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:03.051862 32195 ts_tablet_manager.cc:1875] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:03.051898 32196 ts_tablet_manager.cc:1875] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:03.052158 32197 ts_tablet_manager.cc:1875] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:03.054064 32195 ts_tablet_manager.cc:1888] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:03.054138 32195 log.cc:1196] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/d0afadaff02a4abbaf34a35f647a8c6c
I20240502 07:59:03.054407 32195 ts_tablet_manager.cc:1909] T d0afadaff02a4abbaf34a35f647a8c6c P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:59:03.054395 32196 ts_tablet_manager.cc:1888] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:03.054489 32196 log.cc:1196] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/d0afadaff02a4abbaf34a35f647a8c6c
I20240502 07:59:03.054862 32196 ts_tablet_manager.cc:1909] T d0afadaff02a4abbaf34a35f647a8c6c P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:59:03.054946 32197 ts_tablet_manager.cc:1888] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:03.055051 32197 log.cc:1196] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/d0afadaff02a4abbaf34a35f647a8c6c
I20240502 07:59:03.055063 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet d0afadaff02a4abbaf34a35f647a8c6c (table rebalancer_dynamic_table_new_name [id=1600c880fda4465789f2d598c33d387f]) successfully deleted
I20240502 07:59:03.055378 32197 ts_tablet_manager.cc:1909] T d0afadaff02a4abbaf34a35f647a8c6c P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:59:03.055550 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet d0afadaff02a4abbaf34a35f647a8c6c (table rebalancer_dynamic_table_new_name [id=1600c880fda4465789f2d598c33d387f]) successfully deleted
I20240502 07:59:03.055964 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet d0afadaff02a4abbaf34a35f647a8c6c (table rebalancer_dynamic_table_new_name [id=1600c880fda4465789f2d598c33d387f]) successfully deleted
W20240502 07:59:03.067384 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20240502 07:59:03.151623 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:59:03.156433 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 31abd39eff354d4b8e9bdcb62a481bd3 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=c64f2fd36f43484eae2c2df88aa049ce]), partition=
I20240502 07:59:03.156548 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 31abd39eff354d4b8e9bdcb62a481bd3 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=c64f2fd36f43484eae2c2df88aa049ce]), partition=
I20240502 07:59:03.156623 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 31abd39eff354d4b8e9bdcb62a481bd3 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=c64f2fd36f43484eae2c2df88aa049ce]), partition=
I20240502 07:59:03.156771 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 31abd39eff354d4b8e9bdcb62a481bd3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:03.156771 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 31abd39eff354d4b8e9bdcb62a481bd3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:03.156904 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 31abd39eff354d4b8e9bdcb62a481bd3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:03.158746 32014 tablet_bootstrap.cc:492] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:03.158771 32144 tablet_bootstrap.cc:492] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:03.159018 32200 tablet_bootstrap.cc:492] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:03.159514 32144 tablet_bootstrap.cc:654] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:03.159544 32014 tablet_bootstrap.cc:654] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:03.159826 32200 tablet_bootstrap.cc:654] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:03.160339 32144 tablet_bootstrap.cc:492] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:03.160418 32144 ts_tablet_manager.cc:1367] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:59:03.160825 32144 raft_consensus.cc:348] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.160923 32144 raft_consensus.cc:374] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:03.160950 32144 raft_consensus.cc:729] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:03.161049 32144 consensus_queue.cc:260] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.161221 32014 tablet_bootstrap.cc:492] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:03.161284 32014 ts_tablet_manager.cc:1367] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.001s
I20240502 07:59:03.161315 32144 ts_tablet_manager.cc:1398] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:03.161736 32014 raft_consensus.cc:348] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.161815 32014 raft_consensus.cc:374] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:03.161834 32014 raft_consensus.cc:729] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:03.161903 32014 consensus_queue.cc:260] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.162117 32014 ts_tablet_manager.cc:1398] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:03.163516 32200 tablet_bootstrap.cc:492] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:03.163595 32200 ts_tablet_manager.cc:1367] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:59:03.165904 32200 raft_consensus.cc:348] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.166045 32200 raft_consensus.cc:374] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:03.166082 32200 raft_consensus.cc:729] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:03.166195 32200 consensus_queue.cc:260] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.166545 32200 ts_tablet_manager.cc:1398] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:59:03.256902 31749 raft_consensus.cc:482] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:03.257005 31749 raft_consensus.cc:504] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.257493 31749 leader_election.cc:288] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:03.257714 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "31abd39eff354d4b8e9bdcb62a481bd3" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:03.257836 25135 raft_consensus.cc:2454] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:59:03.257815 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "31abd39eff354d4b8e9bdcb62a481bd3" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:03.257933 22979 raft_consensus.cc:2454] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:59:03.258080 22653 leader_election.cc:302] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:03.258232 31749 raft_consensus.cc:2781] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:03.258285 31749 raft_consensus.cc:482] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:03.258308 31749 raft_consensus.cc:3037] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:03.259146 31749 raft_consensus.cc:504] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.259413 31749 leader_election.cc:288] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:03.259651 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "31abd39eff354d4b8e9bdcb62a481bd3" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:03.259662 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "31abd39eff354d4b8e9bdcb62a481bd3" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:03.259737 25135 raft_consensus.cc:3037] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:03.259754 22979 raft_consensus.cc:3037] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:03.260558 25135 raft_consensus.cc:2454] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:59:03.260768 22979 raft_consensus.cc:2454] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:59:03.260818 22653 leader_election.cc:302] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:03.260968 31749 raft_consensus.cc:2781] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:03.261176 31749 raft_consensus.cc:686] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:59:03.261288 31749 consensus_queue.cc:237] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.263142 22444 catalog_manager.cc:5484] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
W20240502 07:59:03.279489 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:03.314821 31793 consensus_queue.cc:1035] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:03.318243 31698 consensus_queue.cc:1035] T 69fad10dadc74d24a783fd05bc36fa9a P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:03.380297 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:03.380703 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:59:03.381201 32026 consensus_queue.cc:1035] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:03.385316 32023 consensus_queue.cc:1035] T 28dcd73d0afc4d36a659bf5fc2daa897 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:03.388432 22979 raft_consensus.cc:1261] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:03.388742 31749 consensus_queue.cc:1035] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:03.389333 25135 raft_consensus.cc:1261] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:03.389760 31749 consensus_queue.cc:1035] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:03.390866 32132 tablet.cc:1722] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:03.391132 32132 tablet.cc:1620] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:59:03.392853 22438 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet 31abd39eff354d4b8e9bdcb62a481bd3 (table rebalancer_dynamic_table [id=c64f2fd36f43484eae2c2df88aa049ce]). Expected version 1 got 0
I20240502 07:59:03.393880 22444 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 31abd39eff354d4b8e9bdcb62a481bd3 (table rebalancer_dynamic_table [id=c64f2fd36f43484eae2c2df88aa049ce]). Expected version 1 got 0
I20240502 07:59:03.395162 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=c64f2fd36f43484eae2c2df88aa049ce] alter complete (version 1)
I20240502 07:59:03.395584 32130 tablet.cc:1722] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:03.395871 32130 tablet.cc:1620] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:03.396010 32248 tablet.cc:1722] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:03.396965 22436 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet 31abd39eff354d4b8e9bdcb62a481bd3 (table rebalancer_dynamic_table [id=c64f2fd36f43484eae2c2df88aa049ce]). Expected version 1 got 0
I20240502 07:59:03.397094 32248 tablet.cc:1620] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:59:03.397460 22438 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 31abd39eff354d4b8e9bdcb62a481bd3 (table rebalancer_dynamic_table [id=c64f2fd36f43484eae2c2df88aa049ce]). Expected version 1 got 0
W20240502 07:59:03.422621 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:03.442023 31698 consensus_queue.cc:1035] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:03.444432 31793 consensus_queue.cc:1035] T d2dfb8781b2f4f7ab1da2c8fb68193fa P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:03.474938 22438 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_56"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:59:03.480309 22959 tablet_service.cc:1467] Processing CreateTablet for tablet eabce8c834bc40168c7f5a5957cb81d2 (DEFAULT_TABLE table=rebalancer_extra_table_56 [id=8d038a8a527649828174ec0cf777d01e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:03.480389 25246 tablet_service.cc:1467] Processing CreateTablet for tablet eabce8c834bc40168c7f5a5957cb81d2 (DEFAULT_TABLE table=rebalancer_extra_table_56 [id=8d038a8a527649828174ec0cf777d01e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:03.480620 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eabce8c834bc40168c7f5a5957cb81d2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:03.480676 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eabce8c834bc40168c7f5a5957cb81d2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:03.481257 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 1ec12e531e6145ada25a369ed15d1a6c (DEFAULT_TABLE table=rebalancer_extra_table_56 [id=8d038a8a527649828174ec0cf777d01e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:03.481484 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1ec12e531e6145ada25a369ed15d1a6c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:03.482267 32200 tablet_bootstrap.cc:492] T eabce8c834bc40168c7f5a5957cb81d2 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:03.483032 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 1ec12e531e6145ada25a369ed15d1a6c (DEFAULT_TABLE table=rebalancer_extra_table_56 [id=8d038a8a527649828174ec0cf777d01e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:03.483211 32200 tablet_bootstrap.cc:654] T eabce8c834bc40168c7f5a5957cb81d2 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:03.483302 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1ec12e531e6145ada25a369ed15d1a6c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:03.483455 22827 tablet_service.cc:1467] Processing CreateTablet for tablet eabce8c834bc40168c7f5a5957cb81d2 (DEFAULT_TABLE table=rebalancer_extra_table_56 [id=8d038a8a527649828174ec0cf777d01e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:03.483709 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eabce8c834bc40168c7f5a5957cb81d2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:03.483752 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 86f6f1d30fba457198ad3445881cfc7f (DEFAULT_TABLE table=rebalancer_extra_table_56 [id=8d038a8a527649828174ec0cf777d01e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:03.484009 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 86f6f1d30fba457198ad3445881cfc7f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:03.484947 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 1ec12e531e6145ada25a369ed15d1a6c (DEFAULT_TABLE table=rebalancer_extra_table_56 [id=8d038a8a527649828174ec0cf777d01e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:03.485210 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1ec12e531e6145ada25a369ed15d1a6c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:03.485531 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 86f6f1d30fba457198ad3445881cfc7f (DEFAULT_TABLE table=rebalancer_extra_table_56 [id=8d038a8a527649828174ec0cf777d01e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:03.485744 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 86f6f1d30fba457198ad3445881cfc7f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:03.485747 32200 tablet_bootstrap.cc:492] T eabce8c834bc40168c7f5a5957cb81d2 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:03.485814 32200 ts_tablet_manager.cc:1367] T eabce8c834bc40168c7f5a5957cb81d2 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:59:03.486058 32144 tablet_bootstrap.cc:492] T 1ec12e531e6145ada25a369ed15d1a6c P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:03.486205 32200 raft_consensus.cc:348] T eabce8c834bc40168c7f5a5957cb81d2 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:03.486294 32200 raft_consensus.cc:374] T eabce8c834bc40168c7f5a5957cb81d2 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:03.486320 32200 raft_consensus.cc:729] T eabce8c834bc40168c7f5a5957cb81d2 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:03.486428 32200 consensus_queue.cc:260] T eabce8c834bc40168c7f5a5957cb81d2 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:03.486685 32200 ts_tablet_manager.cc:1398] T eabce8c834bc40168c7f5a5957cb81d2 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:03.486897 32144 tablet_bootstrap.cc:654] T 1ec12e531e6145ada25a369ed15d1a6c P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:03.488781 32286 tablet_bootstrap.cc:492] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:03.489403 32144 tablet_bootstrap.cc:492] T 1ec12e531e6145ada25a369ed15d1a6c P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:03.489485 32144 ts_tablet_manager.cc:1367] T 1ec12e531e6145ada25a369ed15d1a6c P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:59:03.489557 32014 tablet_bootstrap.cc:492] T 86f6f1d30fba457198ad3445881cfc7f P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:03.489931 32286 tablet_bootstrap.cc:654] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:03.489881 32144 raft_consensus.cc:348] T 1ec12e531e6145ada25a369ed15d1a6c P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:03.489971 32144 raft_consensus.cc:374] T 1ec12e531e6145ada25a369ed15d1a6c P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:03.489998 32144 raft_consensus.cc:729] T 1ec12e531e6145ada25a369ed15d1a6c P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:03.490106 32144 consensus_queue.cc:260] T 1ec12e531e6145ada25a369ed15d1a6c P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:03.490343 32144 ts_tablet_manager.cc:1398] T 1ec12e531e6145ada25a369ed15d1a6c P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:03.490398 32014 tablet_bootstrap.cc:654] T 86f6f1d30fba457198ad3445881cfc7f P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:03.490406 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 86f6f1d30fba457198ad3445881cfc7f (DEFAULT_TABLE table=rebalancer_extra_table_56 [id=8d038a8a527649828174ec0cf777d01e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:03.490599 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 86f6f1d30fba457198ad3445881cfc7f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:03.490621 32283 tablet_bootstrap.cc:492] T eabce8c834bc40168c7f5a5957cb81d2 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:03.492142 32287 tablet_bootstrap.cc:492] T 1ec12e531e6145ada25a369ed15d1a6c P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:03.492476 32200 tablet_bootstrap.cc:492] T 86f6f1d30fba457198ad3445881cfc7f P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:03.493266 32200 tablet_bootstrap.cc:654] T 86f6f1d30fba457198ad3445881cfc7f P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:03.493570 32284 tablet_bootstrap.cc:492] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:03.494109 32014 tablet_bootstrap.cc:492] T 86f6f1d30fba457198ad3445881cfc7f P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:03.494185 32014 ts_tablet_manager.cc:1367] T 86f6f1d30fba457198ad3445881cfc7f P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.002s
I20240502 07:59:03.494644 32014 raft_consensus.cc:348] T 86f6f1d30fba457198ad3445881cfc7f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.494761 32014 raft_consensus.cc:374] T 86f6f1d30fba457198ad3445881cfc7f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:03.494798 32014 raft_consensus.cc:729] T 86f6f1d30fba457198ad3445881cfc7f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:03.494843 32284 tablet_bootstrap.cc:654] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:03.494912 32014 consensus_queue.cc:260] T 86f6f1d30fba457198ad3445881cfc7f P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.495193 32014 ts_tablet_manager.cc:1398] T 86f6f1d30fba457198ad3445881cfc7f P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:03.496268 32286 tablet_bootstrap.cc:492] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:03.496387 32286 ts_tablet_manager.cc:1367] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.000s
I20240502 07:59:03.496501 32287 tablet_bootstrap.cc:654] T 1ec12e531e6145ada25a369ed15d1a6c P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:03.496834 32286 raft_consensus.cc:348] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:03.497018 32286 raft_consensus.cc:374] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:03.497099 32286 raft_consensus.cc:729] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:03.497330 32286 consensus_queue.cc:260] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:03.497697 32286 ts_tablet_manager.cc:1398] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:03.498785 32200 tablet_bootstrap.cc:492] T 86f6f1d30fba457198ad3445881cfc7f P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:03.498860 32200 ts_tablet_manager.cc:1367] T 86f6f1d30fba457198ad3445881cfc7f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:59:03.498981 32284 tablet_bootstrap.cc:492] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:03.499049 32284 ts_tablet_manager.cc:1367] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:59:03.499274 32200 raft_consensus.cc:348] T 86f6f1d30fba457198ad3445881cfc7f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.499354 32200 raft_consensus.cc:374] T 86f6f1d30fba457198ad3445881cfc7f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:03.499351 32283 tablet_bootstrap.cc:654] T eabce8c834bc40168c7f5a5957cb81d2 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:03.499383 32200 raft_consensus.cc:729] T 86f6f1d30fba457198ad3445881cfc7f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:03.499455 32284 raft_consensus.cc:348] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:03.499485 32200 consensus_queue.cc:260] T 86f6f1d30fba457198ad3445881cfc7f P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.499537 32284 raft_consensus.cc:374] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:03.499562 32284 raft_consensus.cc:729] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:03.499668 32284 consensus_queue.cc:260] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:03.499751 32200 ts_tablet_manager.cc:1398] T 86f6f1d30fba457198ad3445881cfc7f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:03.500073 22438 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
W20240502 07:59:03.500594 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:03.500981 32284 ts_tablet_manager.cc:1398] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:03.503463 32287 tablet_bootstrap.cc:492] T 1ec12e531e6145ada25a369ed15d1a6c P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:03.503604 32287 ts_tablet_manager.cc:1367] T 1ec12e531e6145ada25a369ed15d1a6c P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.012s user 0.002s sys 0.000s
I20240502 07:59:03.504370 32287 raft_consensus.cc:348] T 1ec12e531e6145ada25a369ed15d1a6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:03.504478 32287 raft_consensus.cc:374] T 1ec12e531e6145ada25a369ed15d1a6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:03.504527 32287 raft_consensus.cc:729] T 1ec12e531e6145ada25a369ed15d1a6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:03.504685 32287 consensus_queue.cc:260] T 1ec12e531e6145ada25a369ed15d1a6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:03.505144 32287 ts_tablet_manager.cc:1398] T 1ec12e531e6145ada25a369ed15d1a6c P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:03.505434 32287 tablet_bootstrap.cc:492] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:03.505987 32283 tablet_bootstrap.cc:492] T eabce8c834bc40168c7f5a5957cb81d2 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:03.506083 32283 ts_tablet_manager.cc:1367] T eabce8c834bc40168c7f5a5957cb81d2 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.016s user 0.002s sys 0.000s
I20240502 07:59:03.506517 32283 raft_consensus.cc:348] T eabce8c834bc40168c7f5a5957cb81d2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:03.506616 32283 raft_consensus.cc:374] T eabce8c834bc40168c7f5a5957cb81d2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:03.506655 32283 raft_consensus.cc:729] T eabce8c834bc40168c7f5a5957cb81d2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:03.506784 32283 consensus_queue.cc:260] T eabce8c834bc40168c7f5a5957cb81d2 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:03.507058 32283 ts_tablet_manager.cc:1398] T eabce8c834bc40168c7f5a5957cb81d2 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:03.509058 32132 tablet.cc:1722] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:03.509335 32132 tablet.cc:1620] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:59:03.509953 32248 tablet.cc:1722] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:03.510202 32248 tablet.cc:1620] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:59:03.512614 32130 tablet.cc:1722] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:03.512910 32130 tablet.cc:1620] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
W20240502 07:59:03.513741 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:03.514169 32287 tablet_bootstrap.cc:654] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:03.515453 32287 tablet_bootstrap.cc:492] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:03.515568 32287 ts_tablet_manager.cc:1367] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.000s
I20240502 07:59:03.516022 32287 raft_consensus.cc:348] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.516108 32287 raft_consensus.cc:374] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:03.516135 32287 raft_consensus.cc:729] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:03.516258 32287 consensus_queue.cc:260] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.516541 32287 ts_tablet_manager.cc:1398] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:03.516836 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=c64f2fd36f43484eae2c2df88aa049ce] alter complete (version 2)
W20240502 07:59:03.529156 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:03.584121 32258 raft_consensus.cc:482] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:03.584224 32258 raft_consensus.cc:504] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.584690 32258 leader_election.cc:288] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:03.584952 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "86f6f1d30fba457198ad3445881cfc7f" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:03.584993 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "86f6f1d30fba457198ad3445881cfc7f" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:03.585081 22716 raft_consensus.cc:2454] T 86f6f1d30fba457198ad3445881cfc7f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:03.585131 22979 raft_consensus.cc:2454] T 86f6f1d30fba457198ad3445881cfc7f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:03.585374 22520 leader_election.cc:302] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:03.585660 32258 raft_consensus.cc:2781] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:03.585726 32258 raft_consensus.cc:482] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:03.585762 32258 raft_consensus.cc:3037] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:03.586776 32258 raft_consensus.cc:504] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.587075 32258 leader_election.cc:288] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:03.587359 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "86f6f1d30fba457198ad3445881cfc7f" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:03.587337 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "86f6f1d30fba457198ad3445881cfc7f" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:03.587443 22716 raft_consensus.cc:3037] T 86f6f1d30fba457198ad3445881cfc7f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:03.587442 22979 raft_consensus.cc:3037] T 86f6f1d30fba457198ad3445881cfc7f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:03.588279 22716 raft_consensus.cc:2454] T 86f6f1d30fba457198ad3445881cfc7f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:03.588279 22979 raft_consensus.cc:2454] T 86f6f1d30fba457198ad3445881cfc7f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:03.588568 22520 leader_election.cc:302] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:03.588749 32171 raft_consensus.cc:2781] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:03.588819 32171 raft_consensus.cc:686] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:59:03.588932 32171 consensus_queue.cc:237] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:03.590840 22438 catalog_manager.cc:5484] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:03.621562 22438 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
W20240502 07:59:03.625064 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:03.626121 32132 tablet.cc:1722] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:03.626591 32130 tablet.cc:1722] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:03.626693 32248 tablet.cc:1722] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:03.628499 32199 raft_consensus.cc:482] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:03.628587 32199 raft_consensus.cc:504] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:03.628960 32199 leader_election.cc:288] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:03.629379 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eabce8c834bc40168c7f5a5957cb81d2" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:03.629516 25265 raft_consensus.cc:2454] T eabce8c834bc40168c7f5a5957cb81d2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:03.629777 22783 leader_election.cc:302] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:03.629987 32199 raft_consensus.cc:2781] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:03.630054 32199 raft_consensus.cc:482] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:03.630033 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eabce8c834bc40168c7f5a5957cb81d2" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:03.630100 32199 raft_consensus.cc:3037] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:03.630131 22978 raft_consensus.cc:2454] T eabce8c834bc40168c7f5a5957cb81d2 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:03.631016 32199 raft_consensus.cc:504] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:03.631404 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=c64f2fd36f43484eae2c2df88aa049ce] alter complete (version 3)
I20240502 07:59:03.631668 32199 leader_election.cc:288] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:03.631784 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eabce8c834bc40168c7f5a5957cb81d2" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:03.631896 25265 raft_consensus.cc:3037] T eabce8c834bc40168c7f5a5957cb81d2 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:03.631942 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eabce8c834bc40168c7f5a5957cb81d2" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:03.632037 22978 raft_consensus.cc:3037] T eabce8c834bc40168c7f5a5957cb81d2 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:03.632853 22978 raft_consensus.cc:2454] T eabce8c834bc40168c7f5a5957cb81d2 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:03.633013 25265 raft_consensus.cc:2454] T eabce8c834bc40168c7f5a5957cb81d2 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:03.633275 22782 leader_election.cc:302] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:03.633850 32198 raft_consensus.cc:2781] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:03.634001 32198 raft_consensus.cc:686] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:59:03.634132 32198 consensus_queue.cc:237] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:03.636128 22444 catalog_manager.cc:5484] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } }
I20240502 07:59:03.651665 32236 raft_consensus.cc:482] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:03.651772 32236 raft_consensus.cc:504] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:03.652060 32236 leader_election.cc:288] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:59:03.652302 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1ec12e531e6145ada25a369ed15d1a6c" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:03.652415 25133 raft_consensus.cc:2454] T 1ec12e531e6145ada25a369ed15d1a6c P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:03.652408 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1ec12e531e6145ada25a369ed15d1a6c" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:03.652504 22584 raft_consensus.cc:2454] T 1ec12e531e6145ada25a369ed15d1a6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:03.652724 24808 leader_election.cc:302] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:03.652902 32236 raft_consensus.cc:2781] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:03.652951 32236 raft_consensus.cc:482] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:03.652997 32236 raft_consensus.cc:3037] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:03.654029 32236 raft_consensus.cc:504] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:03.654331 32236 leader_election.cc:288] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:59:03.654760 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1ec12e531e6145ada25a369ed15d1a6c" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:03.654810 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1ec12e531e6145ada25a369ed15d1a6c" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:03.654875 25133 raft_consensus.cc:3037] T 1ec12e531e6145ada25a369ed15d1a6c P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:03.654903 22584 raft_consensus.cc:3037] T 1ec12e531e6145ada25a369ed15d1a6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:03.655700 25133 raft_consensus.cc:2454] T 1ec12e531e6145ada25a369ed15d1a6c P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:03.655860 22584 raft_consensus.cc:2454] T 1ec12e531e6145ada25a369ed15d1a6c P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:03.656086 24808 leader_election.cc:302] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:03.656347 32236 raft_consensus.cc:2781] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:03.656446 32236 raft_consensus.cc:686] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:59:03.656576 32236 consensus_queue.cc:237] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:03.658886 22444 catalog_manager.cc:5484] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
W20240502 07:59:03.710695 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:03.737759 22444 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:03.737936 22444 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:03.740068 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 31abd39eff354d4b8e9bdcb62a481bd3
I20240502 07:59:03.740585 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet 31abd39eff354d4b8e9bdcb62a481bd3 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:03 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:59:03.740936 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet 31abd39eff354d4b8e9bdcb62a481bd3 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:03 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:59:03.741147 25113 tablet_service.cc:1514] Processing DeleteTablet for tablet 31abd39eff354d4b8e9bdcb62a481bd3 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:03 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:59:03.741339 32332 tablet_replica.cc:330] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:59:03.741475 32332 raft_consensus.cc:2229] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:59:03.741657 32332 raft_consensus.cc:2258] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:03.741765 32333 tablet_replica.cc:330] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:59:03.741936 32333 raft_consensus.cc:2229] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:03.742013 32334 tablet_replica.cc:330] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:59:03.742055 32333 raft_consensus.cc:2258] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:03.742154 32334 raft_consensus.cc:2229] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:03.742264 32334 raft_consensus.cc:2258] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:03.742682 32333 ts_tablet_manager.cc:1875] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:03.742729 32334 ts_tablet_manager.cc:1875] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:03.744728 32334 ts_tablet_manager.cc:1888] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:03.744727 32333 ts_tablet_manager.cc:1888] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:03.744803 32333 log.cc:1196] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/31abd39eff354d4b8e9bdcb62a481bd3
I20240502 07:59:03.744803 32334 log.cc:1196] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/31abd39eff354d4b8e9bdcb62a481bd3
I20240502 07:59:03.745095 32334 ts_tablet_manager.cc:1909] T 31abd39eff354d4b8e9bdcb62a481bd3 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:59:03.745117 32333 ts_tablet_manager.cc:1909] T 31abd39eff354d4b8e9bdcb62a481bd3 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:59:03.745720 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 31abd39eff354d4b8e9bdcb62a481bd3 (table rebalancer_dynamic_table_new_name [id=c64f2fd36f43484eae2c2df88aa049ce]) successfully deleted
I20240502 07:59:03.745743 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet 31abd39eff354d4b8e9bdcb62a481bd3 (table rebalancer_dynamic_table_new_name [id=c64f2fd36f43484eae2c2df88aa049ce]) successfully deleted
I20240502 07:59:03.745869 32332 ts_tablet_manager.cc:1875] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:03.747406 32332 ts_tablet_manager.cc:1888] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:03.747464 32332 log.cc:1196] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/31abd39eff354d4b8e9bdcb62a481bd3
I20240502 07:59:03.747720 32332 ts_tablet_manager.cc:1909] T 31abd39eff354d4b8e9bdcb62a481bd3 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:59:03.748201 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 31abd39eff354d4b8e9bdcb62a481bd3 (table rebalancer_dynamic_table_new_name [id=c64f2fd36f43484eae2c2df88aa049ce]) successfully deleted
W20240502 07:59:03.769685 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 07:59:03.793701 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:03.841792 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
W20240502 07:59:03.843447 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:03.847568 22959 tablet_service.cc:1467] Processing CreateTablet for tablet d10cb5fb1efd429282d77da016d9192f (DEFAULT_TABLE table=rebalancer_dynamic_table [id=de21bb8470dd4754982284faeb66ac2c]), partition=
I20240502 07:59:03.847661 24852 tablet_service.cc:1467] Processing CreateTablet for tablet d10cb5fb1efd429282d77da016d9192f (DEFAULT_TABLE table=rebalancer_dynamic_table [id=de21bb8470dd4754982284faeb66ac2c]), partition=
I20240502 07:59:03.847815 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d10cb5fb1efd429282d77da016d9192f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:03.847926 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d10cb5fb1efd429282d77da016d9192f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:03.849735 32200 tablet_bootstrap.cc:492] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:03.849890 32284 tablet_bootstrap.cc:492] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:03.850488 32200 tablet_bootstrap.cc:654] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:03.850834 32284 tablet_bootstrap.cc:654] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:03.851487 32200 tablet_bootstrap.cc:492] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:03.851584 32200 ts_tablet_manager.cc:1367] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:03.852034 22697 tablet_service.cc:1467] Processing CreateTablet for tablet d10cb5fb1efd429282d77da016d9192f (DEFAULT_TABLE table=rebalancer_dynamic_table [id=de21bb8470dd4754982284faeb66ac2c]), partition=
I20240502 07:59:03.851992 32200 raft_consensus.cc:348] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:03.852105 32200 raft_consensus.cc:374] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:03.852125 32284 tablet_bootstrap.cc:492] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:03.852131 32200 raft_consensus.cc:729] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:03.852178 32284 ts_tablet_manager.cc:1367] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:03.852247 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d10cb5fb1efd429282d77da016d9192f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:03.852241 32200 consensus_queue.cc:260] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:03.852480 32200 ts_tablet_manager.cc:1398] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:03.852509 32284 raft_consensus.cc:348] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:03.852586 32284 raft_consensus.cc:374] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:03.852603 32284 raft_consensus.cc:729] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:03.852722 32284 consensus_queue.cc:260] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:03.852988 32284 ts_tablet_manager.cc:1398] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:03.854977 32014 tablet_bootstrap.cc:492] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:03.855851 32014 tablet_bootstrap.cc:654] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:03.856773 32014 tablet_bootstrap.cc:492] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:03.856846 32014 ts_tablet_manager.cc:1367] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:59:03.857200 32014 raft_consensus.cc:348] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:03.857314 32014 raft_consensus.cc:374] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:03.857348 32014 raft_consensus.cc:729] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:03.857471 32014 consensus_queue.cc:260] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:03.857782 32014 ts_tablet_manager.cc:1398] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:04.038182 32236 raft_consensus.cc:482] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:04.038321 32236 raft_consensus.cc:504] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.038697 32236 leader_election.cc:288] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:04.039067 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d10cb5fb1efd429282d77da016d9192f" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:04.039191 22978 raft_consensus.cc:2454] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:04.039151 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d10cb5fb1efd429282d77da016d9192f" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:04.039271 22716 raft_consensus.cc:2454] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:04.039477 24806 leader_election.cc:302] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:04.039623 32236 raft_consensus.cc:2781] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:04.039675 32236 raft_consensus.cc:482] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:04.039716 32236 raft_consensus.cc:3037] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.040484 32236 raft_consensus.cc:504] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.040829 32236 leader_election.cc:288] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:04.041054 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d10cb5fb1efd429282d77da016d9192f" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:04.041079 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d10cb5fb1efd429282d77da016d9192f" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:04.041153 22978 raft_consensus.cc:3037] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.041170 22716 raft_consensus.cc:3037] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.041828 22978 raft_consensus.cc:2454] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:04.041893 22716 raft_consensus.cc:2454] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:04.042089 24806 leader_election.cc:302] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:04.042225 32236 raft_consensus.cc:2781] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:04.042656 32236 raft_consensus.cc:686] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:59:04.042793 32236 consensus_queue.cc:237] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.046044 22444 catalog_manager.cc:5484] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:04.077323 32198 consensus_queue.cc:1035] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:04.079924 32199 consensus_queue.cc:1035] T eabce8c834bc40168c7f5a5957cb81d2 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:04.104372 32258 consensus_queue.cc:1035] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:04.107213 32377 consensus_queue.cc:1035] T 86f6f1d30fba457198ad3445881cfc7f P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:04.110138 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 07:59:04.154891 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:04.177810 32236 consensus_queue.cc:1035] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:04.181756 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_58"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:59:04.183046 32236 consensus_queue.cc:1035] T 1ec12e531e6145ada25a369ed15d1a6c P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:04.184123 22438 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:59:04.187444 22716 raft_consensus.cc:1261] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:04.187739 22978 raft_consensus.cc:1261] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:04.188225 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 1e08f53f34b3434abe2550a02f3356eb (DEFAULT_TABLE table=rebalancer_extra_table_58 [id=9cfe1d7c2fd94731b5ab4ea9782095d5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.188508 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1e08f53f34b3434abe2550a02f3356eb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.188621 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 1e08f53f34b3434abe2550a02f3356eb (DEFAULT_TABLE table=rebalancer_extra_table_58 [id=9cfe1d7c2fd94731b5ab4ea9782095d5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.188868 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1e08f53f34b3434abe2550a02f3356eb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.189189 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 1e08f53f34b3434abe2550a02f3356eb (DEFAULT_TABLE table=rebalancer_extra_table_58 [id=9cfe1d7c2fd94731b5ab4ea9782095d5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.189390 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1e08f53f34b3434abe2550a02f3356eb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.190721 32284 tablet_bootstrap.cc:492] T 1e08f53f34b3434abe2550a02f3356eb P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:04.191097 32416 tablet_bootstrap.cc:492] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:04.191143 22826 tablet_service.cc:1467] Processing CreateTablet for tablet a7086910436b41e7b52e863d26cf63d8 (DEFAULT_TABLE table=rebalancer_extra_table_58 [id=9cfe1d7c2fd94731b5ab4ea9782095d5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.191287 22697 tablet_service.cc:1467] Processing CreateTablet for tablet a7086910436b41e7b52e863d26cf63d8 (DEFAULT_TABLE table=rebalancer_extra_table_58 [id=9cfe1d7c2fd94731b5ab4ea9782095d5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.191362 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a7086910436b41e7b52e863d26cf63d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.191532 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a7086910436b41e7b52e863d26cf63d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.191649 32284 tablet_bootstrap.cc:654] T 1e08f53f34b3434abe2550a02f3356eb P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.192075 32416 tablet_bootstrap.cc:654] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.193439 32014 tablet_bootstrap.cc:492] T a7086910436b41e7b52e863d26cf63d8 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:04.193480 22564 tablet_service.cc:1467] Processing CreateTablet for tablet a7086910436b41e7b52e863d26cf63d8 (DEFAULT_TABLE table=rebalancer_extra_table_58 [id=9cfe1d7c2fd94731b5ab4ea9782095d5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.193720 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a7086910436b41e7b52e863d26cf63d8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.194273 32014 tablet_bootstrap.cc:654] T a7086910436b41e7b52e863d26cf63d8 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.195142 32284 tablet_bootstrap.cc:492] T 1e08f53f34b3434abe2550a02f3356eb P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:04.195211 32284 ts_tablet_manager.cc:1367] T 1e08f53f34b3434abe2550a02f3356eb P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.001s
I20240502 07:59:04.195668 32284 raft_consensus.cc:348] T 1e08f53f34b3434abe2550a02f3356eb P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:04.195791 32284 raft_consensus.cc:374] T 1e08f53f34b3434abe2550a02f3356eb P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.195837 32284 raft_consensus.cc:729] T 1e08f53f34b3434abe2550a02f3356eb P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.195928 32284 consensus_queue.cc:260] T 1e08f53f34b3434abe2550a02f3356eb P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:04.196267 32284 ts_tablet_manager.cc:1398] T 1e08f53f34b3434abe2550a02f3356eb P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:04.196288 32416 tablet_bootstrap.cc:492] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:04.196360 32416 ts_tablet_manager.cc:1367] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:59:04.196787 32421 consensus_queue.cc:1035] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:04.196810 32416 raft_consensus.cc:348] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:04.196921 32416 raft_consensus.cc:374] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.196951 32416 raft_consensus.cc:729] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.197057 32416 consensus_queue.cc:260] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:04.197086 32421 consensus_queue.cc:1035] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:04.199281 32416 ts_tablet_manager.cc:1398] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:59:04.199504 32416 tablet_bootstrap.cc:492] T a7086910436b41e7b52e863d26cf63d8 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:04.200177 22436 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet d10cb5fb1efd429282d77da016d9192f (table rebalancer_dynamic_table [id=de21bb8470dd4754982284faeb66ac2c]). Expected version 1 got 0
I20240502 07:59:04.200771 24852 tablet_service.cc:1467] Processing CreateTablet for tablet b5813fd227fc4772bf523e11d57da718 (DEFAULT_TABLE table=rebalancer_extra_table_58 [id=9cfe1d7c2fd94731b5ab4ea9782095d5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.200970 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b5813fd227fc4772bf523e11d57da718. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.201268 32416 tablet_bootstrap.cc:654] T a7086910436b41e7b52e863d26cf63d8 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.202039 32416 tablet_bootstrap.cc:492] T a7086910436b41e7b52e863d26cf63d8 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:04.202126 32416 ts_tablet_manager.cc:1367] T a7086910436b41e7b52e863d26cf63d8 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.001s
I20240502 07:59:04.202411 32416 raft_consensus.cc:348] T a7086910436b41e7b52e863d26cf63d8 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.202489 32416 raft_consensus.cc:374] T a7086910436b41e7b52e863d26cf63d8 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.202518 32416 raft_consensus.cc:729] T a7086910436b41e7b52e863d26cf63d8 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.202648 32416 consensus_queue.cc:260] T a7086910436b41e7b52e863d26cf63d8 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.202814 32284 tablet_bootstrap.cc:492] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:04.203519 32416 ts_tablet_manager.cc:1398] T a7086910436b41e7b52e863d26cf63d8 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:04.203610 32284 tablet_bootstrap.cc:654] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.203852 22438 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet d10cb5fb1efd429282d77da016d9192f (table rebalancer_dynamic_table [id=de21bb8470dd4754982284faeb66ac2c]). Expected version 1 got 0
I20240502 07:59:04.204583 32284 tablet_bootstrap.cc:492] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:04.204689 32284 ts_tablet_manager.cc:1367] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:59:04.204950 25246 tablet_service.cc:1467] Processing CreateTablet for tablet b5813fd227fc4772bf523e11d57da718 (DEFAULT_TABLE table=rebalancer_extra_table_58 [id=9cfe1d7c2fd94731b5ab4ea9782095d5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.205107 32284 raft_consensus.cc:348] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:04.205171 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b5813fd227fc4772bf523e11d57da718. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.205194 32284 raft_consensus.cc:374] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.205227 32284 raft_consensus.cc:729] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.205343 32284 consensus_queue.cc:260] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:04.205596 32284 ts_tablet_manager.cc:1398] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:04.205894 32415 tablet_bootstrap.cc:492] T 1e08f53f34b3434abe2550a02f3356eb P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:04.206070 32420 tablet_bootstrap.cc:492] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:04.207701 32415 tablet_bootstrap.cc:654] T 1e08f53f34b3434abe2550a02f3356eb P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.208500 32420 tablet_bootstrap.cc:654] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.209681 32014 tablet_bootstrap.cc:492] T a7086910436b41e7b52e863d26cf63d8 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:04.209786 32014 ts_tablet_manager.cc:1367] T a7086910436b41e7b52e863d26cf63d8 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.016s user 0.000s sys 0.002s
I20240502 07:59:04.210361 22438 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet d10cb5fb1efd429282d77da016d9192f (table rebalancer_dynamic_table [id=de21bb8470dd4754982284faeb66ac2c]). Expected version 1 got 0
I20240502 07:59:04.210327 32014 raft_consensus.cc:348] T a7086910436b41e7b52e863d26cf63d8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.210433 32014 raft_consensus.cc:374] T a7086910436b41e7b52e863d26cf63d8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.210398 25113 tablet_service.cc:1467] Processing CreateTablet for tablet b5813fd227fc4772bf523e11d57da718 (DEFAULT_TABLE table=rebalancer_extra_table_58 [id=9cfe1d7c2fd94731b5ab4ea9782095d5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.210469 32014 raft_consensus.cc:729] T a7086910436b41e7b52e863d26cf63d8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.210655 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b5813fd227fc4772bf523e11d57da718. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.210587 32014 consensus_queue.cc:260] T a7086910436b41e7b52e863d26cf63d8 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.210911 32014 ts_tablet_manager.cc:1398] T a7086910436b41e7b52e863d26cf63d8 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:04.214772 32427 tablet_bootstrap.cc:492] T b5813fd227fc4772bf523e11d57da718 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:04.215159 22436 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet d10cb5fb1efd429282d77da016d9192f (table rebalancer_dynamic_table [id=de21bb8470dd4754982284faeb66ac2c]). Expected version 1 got 0
I20240502 07:59:04.215912 32427 tablet_bootstrap.cc:654] T b5813fd227fc4772bf523e11d57da718 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.216907 32427 tablet_bootstrap.cc:492] T b5813fd227fc4772bf523e11d57da718 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:04.216928 32415 tablet_bootstrap.cc:492] T 1e08f53f34b3434abe2550a02f3356eb P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:04.216995 32415 ts_tablet_manager.cc:1367] T 1e08f53f34b3434abe2550a02f3356eb P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.000s
I20240502 07:59:04.217018 32427 ts_tablet_manager.cc:1367] T b5813fd227fc4772bf523e11d57da718 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:04.217379 32427 raft_consensus.cc:348] T b5813fd227fc4772bf523e11d57da718 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:04.217407 32415 raft_consensus.cc:348] T 1e08f53f34b3434abe2550a02f3356eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:04.217471 32427 raft_consensus.cc:374] T b5813fd227fc4772bf523e11d57da718 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.217483 32415 raft_consensus.cc:374] T 1e08f53f34b3434abe2550a02f3356eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.217497 32427 raft_consensus.cc:729] T b5813fd227fc4772bf523e11d57da718 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.217523 32415 raft_consensus.cc:729] T 1e08f53f34b3434abe2550a02f3356eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.217605 32427 consensus_queue.cc:260] T b5813fd227fc4772bf523e11d57da718 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:04.217621 32415 consensus_queue.cc:260] T 1e08f53f34b3434abe2550a02f3356eb P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:04.217856 32427 ts_tablet_manager.cc:1398] T b5813fd227fc4772bf523e11d57da718 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:04.220099 32415 ts_tablet_manager.cc:1398] T 1e08f53f34b3434abe2550a02f3356eb P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:59:04.220292 32415 tablet_bootstrap.cc:492] T b5813fd227fc4772bf523e11d57da718 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:04.221107 32415 tablet_bootstrap.cc:654] T b5813fd227fc4772bf523e11d57da718 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.223251 32415 tablet_bootstrap.cc:492] T b5813fd227fc4772bf523e11d57da718 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:04.223371 32415 ts_tablet_manager.cc:1367] T b5813fd227fc4772bf523e11d57da718 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:59:04.224010 32415 raft_consensus.cc:348] T b5813fd227fc4772bf523e11d57da718 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:04.224094 32415 raft_consensus.cc:374] T b5813fd227fc4772bf523e11d57da718 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.224133 32415 raft_consensus.cc:729] T b5813fd227fc4772bf523e11d57da718 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.224251 32415 consensus_queue.cc:260] T b5813fd227fc4772bf523e11d57da718 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:04.224478 32415 ts_tablet_manager.cc:1398] T b5813fd227fc4772bf523e11d57da718 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:04.224817 32431 tablet.cc:1722] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:04.225173 32431 tablet.cc:1620] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:59:04.227541 22444 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet d10cb5fb1efd429282d77da016d9192f (table rebalancer_dynamic_table [id=de21bb8470dd4754982284faeb66ac2c]). Expected version 1 got 0
I20240502 07:59:04.227572 32428 tablet.cc:1722] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:04.228238 32428 tablet.cc:1620] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:59:04.228680 32434 tablet.cc:1722] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:04.228991 32434 tablet.cc:1620] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:04.233491 32420 tablet_bootstrap.cc:492] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:04.233594 32420 ts_tablet_manager.cc:1367] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.028s user 0.002s sys 0.000s
I20240502 07:59:04.234097 32420 raft_consensus.cc:348] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.234230 32420 raft_consensus.cc:374] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.234728 22440 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet d10cb5fb1efd429282d77da016d9192f (table rebalancer_dynamic_table [id=de21bb8470dd4754982284faeb66ac2c]). Expected version 1 got 0
I20240502 07:59:04.235308 32420 raft_consensus.cc:729] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.235502 32420 consensus_queue.cc:260] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.235790 32420 ts_tablet_manager.cc:1398] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:04.237087 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=de21bb8470dd4754982284faeb66ac2c] alter complete (version 1)
I20240502 07:59:04.277985 32421 raft_consensus.cc:482] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:04.278118 32421 raft_consensus.cc:504] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:04.278553 32421 leader_election.cc:288] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:04.278810 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b5813fd227fc4772bf523e11d57da718" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:04.278944 25133 raft_consensus.cc:2454] T b5813fd227fc4772bf523e11d57da718 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:04.279101 32377 raft_consensus.cc:482] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:04.279204 32377 raft_consensus.cc:504] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.279289 24808 leader_election.cc:302] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:04.279412 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b5813fd227fc4772bf523e11d57da718" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:04.279446 32421 raft_consensus.cc:2781] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:04.279492 32421 raft_consensus.cc:482] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:04.279532 32421 raft_consensus.cc:3037] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.279522 25265 raft_consensus.cc:2454] T b5813fd227fc4772bf523e11d57da718 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:04.279700 32377 leader_election.cc:288] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:04.280062 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a7086910436b41e7b52e863d26cf63d8" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:04.280184 22716 raft_consensus.cc:2454] T a7086910436b41e7b52e863d26cf63d8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:04.280426 22519 leader_election.cc:302] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:04.280625 32377 raft_consensus.cc:2781] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:04.280695 32377 raft_consensus.cc:482] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:04.280733 32377 raft_consensus.cc:3037] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.280694 32421 raft_consensus.cc:504] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:04.281010 32421 leader_election.cc:288] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:04.281221 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b5813fd227fc4772bf523e11d57da718" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:04.281315 25133 raft_consensus.cc:3037] T b5813fd227fc4772bf523e11d57da718 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.281589 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b5813fd227fc4772bf523e11d57da718" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:04.281697 25265 raft_consensus.cc:3037] T b5813fd227fc4772bf523e11d57da718 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.281855 32377 raft_consensus.cc:504] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.282171 32377 leader_election.cc:288] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:04.282263 25133 raft_consensus.cc:2454] T b5813fd227fc4772bf523e11d57da718 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:04.282567 24808 leader_election.cc:302] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:04.282569 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a7086910436b41e7b52e863d26cf63d8" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:59:04.282709 22845 raft_consensus.cc:3037] T a7086910436b41e7b52e863d26cf63d8 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.282712 25265 raft_consensus.cc:2454] T b5813fd227fc4772bf523e11d57da718 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:04.282729 32421 raft_consensus.cc:2781] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:04.282792 32421 raft_consensus.cc:686] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:59:04.282905 32421 consensus_queue.cc:237] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:04.282986 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a7086910436b41e7b52e863d26cf63d8" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:04.283707 22845 raft_consensus.cc:2454] T a7086910436b41e7b52e863d26cf63d8 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:04.284034 22522 leader_election.cc:302] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:04.284999 32377 raft_consensus.cc:2781] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:04.285001 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a7086910436b41e7b52e863d26cf63d8" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:04.285102 22717 raft_consensus.cc:3037] T a7086910436b41e7b52e863d26cf63d8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.285326 32377 raft_consensus.cc:686] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:59:04.285461 32377 consensus_queue.cc:237] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
W20240502 07:59:04.285830 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:04.286063 22717 raft_consensus.cc:2454] T a7086910436b41e7b52e863d26cf63d8 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:04.286123 22440 catalog_manager.cc:5484] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } }
I20240502 07:59:04.288476 22444 catalog_manager.cc:5484] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:04.310400 32199 raft_consensus.cc:482] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:04.310508 32199 raft_consensus.cc:504] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:04.310882 32199 leader_election.cc:288] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:04.311151 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1e08f53f34b3434abe2550a02f3356eb" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:04.311170 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1e08f53f34b3434abe2550a02f3356eb" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:04.311272 24870 raft_consensus.cc:2454] T 1e08f53f34b3434abe2550a02f3356eb P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:04.311276 25133 raft_consensus.cc:2454] T 1e08f53f34b3434abe2550a02f3356eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:04.311589 22784 leader_election.cc:302] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:04.311800 32199 raft_consensus.cc:2781] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:04.311859 32199 raft_consensus.cc:482] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:04.311903 32199 raft_consensus.cc:3037] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.312974 32199 raft_consensus.cc:504] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
W20240502 07:59:04.313290 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:04.313304 32199 leader_election.cc:288] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:04.313582 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1e08f53f34b3434abe2550a02f3356eb" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:04.313679 25133 raft_consensus.cc:3037] T 1e08f53f34b3434abe2550a02f3356eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.313683 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1e08f53f34b3434abe2550a02f3356eb" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:04.313786 24870 raft_consensus.cc:3037] T 1e08f53f34b3434abe2550a02f3356eb P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.314458 25133 raft_consensus.cc:2454] T 1e08f53f34b3434abe2550a02f3356eb P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:04.314608 24870 raft_consensus.cc:2454] T 1e08f53f34b3434abe2550a02f3356eb P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:04.314859 22781 leader_election.cc:302] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:04.315034 32199 raft_consensus.cc:2781] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:04.315131 32199 raft_consensus.cc:686] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:59:04.315271 32199 consensus_queue.cc:237] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:04.317862 22444 catalog_manager.cc:5484] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:04.339169 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:59:04.343605 32428 tablet.cc:1722] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:04.343915 32428 tablet.cc:1620] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:59:04.344092 32431 tablet.cc:1722] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:04.344379 32431 tablet.cc:1620] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:59:04.346264 32434 tablet.cc:1722] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:04.346551 32434 tablet.cc:1620] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:04.350054 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=de21bb8470dd4754982284faeb66ac2c] alter complete (version 2)
W20240502 07:59:04.370396 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:04.451633 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:59:04.457708 32428 tablet.cc:1722] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:04.458176 32434 tablet.cc:1722] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:04.459975 32431 tablet.cc:1722] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:04.464073 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=de21bb8470dd4754982284faeb66ac2c] alter complete (version 3)
I20240502 07:59:04.566737 22436 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:04.566882 22436 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:04.568701 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet d10cb5fb1efd429282d77da016d9192f
I20240502 07:59:04.569195 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet d10cb5fb1efd429282d77da016d9192f with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:04 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:59:04.569231 22697 tablet_service.cc:1514] Processing DeleteTablet for tablet d10cb5fb1efd429282d77da016d9192f with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:04 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:59:04.569408 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet d10cb5fb1efd429282d77da016d9192f with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:04 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:59:04.569661 32471 tablet_replica.cc:330] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:59:04.569746 32470 tablet_replica.cc:330] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:59:04.569885 32470 raft_consensus.cc:2229] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:04.570060 32470 raft_consensus.cc:2258] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:04.570114 32471 raft_consensus.cc:2229] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:59:04.570477 32469 tablet_replica.cc:330] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:59:04.570680 32471 raft_consensus.cc:2258] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:04.571133 32469 raft_consensus.cc:2229] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:04.571297 32469 raft_consensus.cc:2258] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:04.571380 32470 ts_tablet_manager.cc:1875] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:04.571379 32471 ts_tablet_manager.cc:1875] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:04.571839 32469 ts_tablet_manager.cc:1875] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:04.573588 32471 ts_tablet_manager.cc:1888] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:04.573686 32470 ts_tablet_manager.cc:1888] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:04.573714 32471 log.cc:1196] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/d10cb5fb1efd429282d77da016d9192f
I20240502 07:59:04.573856 32470 log.cc:1196] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/d10cb5fb1efd429282d77da016d9192f
I20240502 07:59:04.574050 32471 ts_tablet_manager.cc:1909] T d10cb5fb1efd429282d77da016d9192f P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:59:04.574277 32470 ts_tablet_manager.cc:1909] T d10cb5fb1efd429282d77da016d9192f P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:59:04.574366 32469 ts_tablet_manager.cc:1888] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:04.574494 32469 log.cc:1196] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/d10cb5fb1efd429282d77da016d9192f
I20240502 07:59:04.574838 32469 ts_tablet_manager.cc:1909] T d10cb5fb1efd429282d77da016d9192f P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:59:04.574844 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet d10cb5fb1efd429282d77da016d9192f (table rebalancer_dynamic_table_new_name [id=de21bb8470dd4754982284faeb66ac2c]) successfully deleted
I20240502 07:59:04.575040 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet d10cb5fb1efd429282d77da016d9192f (table rebalancer_dynamic_table_new_name [id=de21bb8470dd4754982284faeb66ac2c]) successfully deleted
I20240502 07:59:04.575533 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet d10cb5fb1efd429282d77da016d9192f (table rebalancer_dynamic_table_new_name [id=de21bb8470dd4754982284faeb66ac2c]) successfully deleted
I20240502 07:59:04.670647 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:59:04.675678 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 849ed56820c54ef892aea88ec91ccc36 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=c1d68ada8f50465e8eca085d5519586b]), partition=
I20240502 07:59:04.675935 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 849ed56820c54ef892aea88ec91ccc36 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=c1d68ada8f50465e8eca085d5519586b]), partition=
I20240502 07:59:04.675971 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 849ed56820c54ef892aea88ec91ccc36. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.676159 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 849ed56820c54ef892aea88ec91ccc36. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.677732 32415 tablet_bootstrap.cc:492] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:04.677894 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 849ed56820c54ef892aea88ec91ccc36 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=c1d68ada8f50465e8eca085d5519586b]), partition=
I20240502 07:59:04.678072 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 849ed56820c54ef892aea88ec91ccc36. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.678592 32415 tablet_bootstrap.cc:654] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.679540 32415 tablet_bootstrap.cc:492] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:04.679620 32415 ts_tablet_manager.cc:1367] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:04.680063 32415 raft_consensus.cc:348] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:04.680157 32415 raft_consensus.cc:374] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.680187 32415 raft_consensus.cc:729] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.680297 32415 consensus_queue.cc:260] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:04.680608 32415 ts_tablet_manager.cc:1398] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:04.680759 32014 tablet_bootstrap.cc:492] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:04.681493 32014 tablet_bootstrap.cc:654] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.682387 32014 tablet_bootstrap.cc:492] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:04.682461 32014 ts_tablet_manager.cc:1367] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:04.682893 32014 raft_consensus.cc:348] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:04.682994 32014 raft_consensus.cc:374] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.683034 32014 raft_consensus.cc:729] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.683143 32014 consensus_queue.cc:260] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:04.683373 32014 ts_tablet_manager.cc:1398] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:04.683636 32503 tablet_bootstrap.cc:492] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:04.684372 32503 tablet_bootstrap.cc:654] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.685770 32503 tablet_bootstrap.cc:492] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:04.685851 32503 ts_tablet_manager.cc:1367] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:04.686230 32503 raft_consensus.cc:348] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:04.686317 32503 raft_consensus.cc:374] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.686345 32503 raft_consensus.cc:729] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.686457 32503 consensus_queue.cc:260] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:04.686714 32503 ts_tablet_manager.cc:1398] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:04.687701 32502 consensus_queue.cc:1035] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:04.692282 32258 consensus_queue.cc:1035] T a7086910436b41e7b52e863d26cf63d8 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:04.693027 32298 consensus_queue.cc:1035] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:04.697700 32198 consensus_queue.cc:1035] T 1e08f53f34b3434abe2550a02f3356eb P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:04.710289 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:04.822381 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:04.841521 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_60"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 07:59:04.848565 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:04.850114 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 5e734db2608e4a01a20d113c8280b559 (DEFAULT_TABLE table=rebalancer_extra_table_60 [id=2c53e9afb5154b47a582dbf219b07f5d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.850165 22826 tablet_service.cc:1467] Processing CreateTablet for tablet 5e734db2608e4a01a20d113c8280b559 (DEFAULT_TABLE table=rebalancer_extra_table_60 [id=2c53e9afb5154b47a582dbf219b07f5d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.850419 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5e734db2608e4a01a20d113c8280b559. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.850421 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5e734db2608e4a01a20d113c8280b559. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.852645 32528 tablet_bootstrap.cc:492] T 5e734db2608e4a01a20d113c8280b559 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:04.852905 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 5e734db2608e4a01a20d113c8280b559 (DEFAULT_TABLE table=rebalancer_extra_table_60 [id=2c53e9afb5154b47a582dbf219b07f5d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.853168 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5e734db2608e4a01a20d113c8280b559. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.853163 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 2e55a187ef9b4eabaa3c4a83c1c3b0da (DEFAULT_TABLE table=rebalancer_extra_table_60 [id=2c53e9afb5154b47a582dbf219b07f5d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.853399 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e55a187ef9b4eabaa3c4a83c1c3b0da. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.853703 32528 tablet_bootstrap.cc:654] T 5e734db2608e4a01a20d113c8280b559 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.854223 32527 tablet_bootstrap.cc:492] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:04.854945 32527 tablet_bootstrap.cc:654] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.855796 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 6d041e9b9503402ba45f5b0dfc777ca5 (DEFAULT_TABLE table=rebalancer_extra_table_60 [id=2c53e9afb5154b47a582dbf219b07f5d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.855937 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 6d041e9b9503402ba45f5b0dfc777ca5 (DEFAULT_TABLE table=rebalancer_extra_table_60 [id=2c53e9afb5154b47a582dbf219b07f5d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.856048 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6d041e9b9503402ba45f5b0dfc777ca5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.856155 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6d041e9b9503402ba45f5b0dfc777ca5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.856195 32530 tablet_bootstrap.cc:492] T 5e734db2608e4a01a20d113c8280b559 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:04.857085 32530 tablet_bootstrap.cc:654] T 5e734db2608e4a01a20d113c8280b559 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.858464 32014 tablet_bootstrap.cc:492] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:04.858531 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 2e55a187ef9b4eabaa3c4a83c1c3b0da (DEFAULT_TABLE table=rebalancer_extra_table_60 [id=2c53e9afb5154b47a582dbf219b07f5d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.858785 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e55a187ef9b4eabaa3c4a83c1c3b0da. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.858803 32530 tablet_bootstrap.cc:492] T 5e734db2608e4a01a20d113c8280b559 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:04.858893 32530 ts_tablet_manager.cc:1367] T 5e734db2608e4a01a20d113c8280b559 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:59:04.859325 32014 tablet_bootstrap.cc:654] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.859341 32530 raft_consensus.cc:348] T 5e734db2608e4a01a20d113c8280b559 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:04.859454 32530 raft_consensus.cc:374] T 5e734db2608e4a01a20d113c8280b559 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.859491 32530 raft_consensus.cc:729] T 5e734db2608e4a01a20d113c8280b559 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.859617 32530 consensus_queue.cc:260] T 5e734db2608e4a01a20d113c8280b559 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:04.859709 32422 raft_consensus.cc:482] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:04.859787 32422 raft_consensus.cc:504] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:04.859922 32530 ts_tablet_manager.cc:1398] T 5e734db2608e4a01a20d113c8280b559 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:04.860131 32422 leader_election.cc:288] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:04.860337 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 6d041e9b9503402ba45f5b0dfc777ca5 (DEFAULT_TABLE table=rebalancer_extra_table_60 [id=2c53e9afb5154b47a582dbf219b07f5d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.860592 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6d041e9b9503402ba45f5b0dfc777ca5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.860623 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "849ed56820c54ef892aea88ec91ccc36" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:04.860785 22979 raft_consensus.cc:2454] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:59:04.861063 32527 tablet_bootstrap.cc:492] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:04.861140 32527 ts_tablet_manager.cc:1367] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 07:59:04.861115 22651 leader_election.cc:302] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:04.861285 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "849ed56820c54ef892aea88ec91ccc36" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:04.861402 25133 raft_consensus.cc:2454] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:59:04.861588 32527 raft_consensus.cc:348] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:04.861699 32527 raft_consensus.cc:374] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.861730 32527 raft_consensus.cc:729] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.861794 32422 raft_consensus.cc:2781] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:04.861855 32422 raft_consensus.cc:482] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:04.861837 32527 consensus_queue.cc:260] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:04.861891 32422 raft_consensus.cc:3037] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.862140 32527 ts_tablet_manager.cc:1398] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:04.862351 32534 tablet_bootstrap.cc:492] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:04.862830 32422 raft_consensus.cc:504] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:04.863176 32422 leader_election.cc:288] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:04.863515 32534 tablet_bootstrap.cc:654] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.864563 32534 tablet_bootstrap.cc:492] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:04.864670 32534 ts_tablet_manager.cc:1367] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:04.865136 32534 raft_consensus.cc:348] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.865229 32534 raft_consensus.cc:374] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.865263 32534 raft_consensus.cc:729] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.865386 32534 consensus_queue.cc:260] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.865670 32534 ts_tablet_manager.cc:1398] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:04.865900 32534 tablet_bootstrap.cc:492] T 6d041e9b9503402ba45f5b0dfc777ca5 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:04.866600 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 2e55a187ef9b4eabaa3c4a83c1c3b0da (DEFAULT_TABLE table=rebalancer_extra_table_60 [id=2c53e9afb5154b47a582dbf219b07f5d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:04.866730 32534 tablet_bootstrap.cc:654] T 6d041e9b9503402ba45f5b0dfc777ca5 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.866847 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e55a187ef9b4eabaa3c4a83c1c3b0da. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:04.869599 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "849ed56820c54ef892aea88ec91ccc36" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:04.869727 25134 raft_consensus.cc:3037] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.870760 32014 tablet_bootstrap.cc:492] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:04.870770 25134 raft_consensus.cc:2454] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:59:04.870851 32014 ts_tablet_manager.cc:1367] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.012s user 0.002s sys 0.000s
I20240502 07:59:04.871280 32014 raft_consensus.cc:348] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.871372 32014 raft_consensus.cc:374] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.871403 32014 raft_consensus.cc:729] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.871511 32014 consensus_queue.cc:260] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.871528 22653 leader_election.cc:302] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:04.871773 32014 ts_tablet_manager.cc:1398] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:04.871755 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "849ed56820c54ef892aea88ec91ccc36" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:04.871870 22979 raft_consensus.cc:3037] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.871963 32014 tablet_bootstrap.cc:492] T 6d041e9b9503402ba45f5b0dfc777ca5 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:04.872860 22979 raft_consensus.cc:2454] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:59:04.872999 32297 raft_consensus.cc:2781] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:04.873067 32297 raft_consensus.cc:686] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:59:04.873189 32297 consensus_queue.cc:237] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:04.875239 32014 tablet_bootstrap.cc:654] T 6d041e9b9503402ba45f5b0dfc777ca5 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.876014 22436 catalog_manager.cc:5484] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:04.876368 32534 tablet_bootstrap.cc:492] T 6d041e9b9503402ba45f5b0dfc777ca5 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:04.876448 32534 ts_tablet_manager.cc:1367] T 6d041e9b9503402ba45f5b0dfc777ca5 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.011s user 0.000s sys 0.002s
I20240502 07:59:04.876535 32014 tablet_bootstrap.cc:492] T 6d041e9b9503402ba45f5b0dfc777ca5 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:04.876519 31935 consensus_queue.cc:1035] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:04.876595 32014 ts_tablet_manager.cc:1367] T 6d041e9b9503402ba45f5b0dfc777ca5 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:59:04.877231 32014 raft_consensus.cc:348] T 6d041e9b9503402ba45f5b0dfc777ca5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:04.877400 32014 raft_consensus.cc:374] T 6d041e9b9503402ba45f5b0dfc777ca5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.877480 32014 raft_consensus.cc:729] T 6d041e9b9503402ba45f5b0dfc777ca5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.877671 32014 consensus_queue.cc:260] T 6d041e9b9503402ba45f5b0dfc777ca5 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:04.878041 32014 ts_tablet_manager.cc:1398] T 6d041e9b9503402ba45f5b0dfc777ca5 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:04.880156 32528 tablet_bootstrap.cc:492] T 5e734db2608e4a01a20d113c8280b559 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:04.880254 32528 ts_tablet_manager.cc:1367] T 5e734db2608e4a01a20d113c8280b559 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.028s user 0.002s sys 0.000s
I20240502 07:59:04.880966 32528 raft_consensus.cc:348] T 5e734db2608e4a01a20d113c8280b559 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:04.881089 32528 raft_consensus.cc:374] T 5e734db2608e4a01a20d113c8280b559 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.881120 32528 raft_consensus.cc:729] T 5e734db2608e4a01a20d113c8280b559 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.881245 32528 consensus_queue.cc:260] T 5e734db2608e4a01a20d113c8280b559 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:04.881539 32528 ts_tablet_manager.cc:1398] T 5e734db2608e4a01a20d113c8280b559 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:04.881784 32528 tablet_bootstrap.cc:492] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:04.882881 32528 tablet_bootstrap.cc:654] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.884222 32421 consensus_queue.cc:1035] T b5813fd227fc4772bf523e11d57da718 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:04.885934 32528 tablet_bootstrap.cc:492] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:04.886013 32528 ts_tablet_manager.cc:1367] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:59:04.886391 32528 raft_consensus.cc:348] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:04.886495 32528 raft_consensus.cc:374] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.886528 32528 raft_consensus.cc:729] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.876916 32534 raft_consensus.cc:348] T 6d041e9b9503402ba45f5b0dfc777ca5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:04.888124 32534 raft_consensus.cc:374] T 6d041e9b9503402ba45f5b0dfc777ca5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.888203 32534 raft_consensus.cc:729] T 6d041e9b9503402ba45f5b0dfc777ca5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.888375 32534 consensus_queue.cc:260] T 6d041e9b9503402ba45f5b0dfc777ca5 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:04.888639 32534 ts_tablet_manager.cc:1398] T 6d041e9b9503402ba45f5b0dfc777ca5 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.012s user 0.001s sys 0.000s
I20240502 07:59:04.888907 32528 consensus_queue.cc:260] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:04.889298 32528 ts_tablet_manager.cc:1398] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:59:04.889464 32528 tablet_bootstrap.cc:492] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:04.894016 32528 tablet_bootstrap.cc:654] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:04.895339 32528 tablet_bootstrap.cc:492] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:04.895427 32528 ts_tablet_manager.cc:1367] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:59:04.895797 32528 raft_consensus.cc:348] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.895874 32528 raft_consensus.cc:374] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:04.895905 32528 raft_consensus.cc:729] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:04.896034 32528 consensus_queue.cc:260] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.896379 32528 ts_tablet_manager.cc:1398] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
W20240502 07:59:04.960328 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 07:59:04.985373 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:04.992318 32297 raft_consensus.cc:482] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:04.992437 32297 raft_consensus.cc:504] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.992944 32297 leader_election.cc:288] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:04.993225 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e55a187ef9b4eabaa3c4a83c1c3b0da" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:04.993252 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e55a187ef9b4eabaa3c4a83c1c3b0da" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:04.993368 25266 raft_consensus.cc:2454] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:59:04.993371 22584 raft_consensus.cc:2454] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:59:04.993615 22653 leader_election.cc:302] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:04.993793 32297 raft_consensus.cc:2781] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:04.993847 32297 raft_consensus.cc:482] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:04.993889 32297 raft_consensus.cc:3037] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.994740 32297 raft_consensus.cc:504] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:04.995033 32297 leader_election.cc:288] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:04.995307 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e55a187ef9b4eabaa3c4a83c1c3b0da" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:04.995405 22584 raft_consensus.cc:3037] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.995359 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e55a187ef9b4eabaa3c4a83c1c3b0da" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:04.995481 25266 raft_consensus.cc:3037] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:04.996186 22584 raft_consensus.cc:2454] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:59:04.996304 25266 raft_consensus.cc:2454] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:59:04.996484 22653 leader_election.cc:302] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:04.996750 32297 raft_consensus.cc:2781] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:04.996835 32297 raft_consensus.cc:686] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:59:04.996968 32297 consensus_queue.cc:237] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
W20240502 07:59:04.998842 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:04.999126 22436 catalog_manager.cc:5484] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } }
I20240502 07:59:05.013159 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:59:05.016224 22978 raft_consensus.cc:1261] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:05.016248 25133 raft_consensus.cc:1261] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:05.016593 32422 consensus_queue.cc:1035] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:05.016737 32297 consensus_queue.cc:1035] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:05.019552 32570 tablet.cc:1722] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:05.019919 32570 tablet.cc:1620] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:05.020288 22436 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet 849ed56820c54ef892aea88ec91ccc36 (table rebalancer_dynamic_table [id=c1d68ada8f50465e8eca085d5519586b]). Expected version 1 got 0
I20240502 07:59:05.021929 22444 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet 849ed56820c54ef892aea88ec91ccc36 (table rebalancer_dynamic_table [id=c1d68ada8f50465e8eca085d5519586b]). Expected version 1 got 0
I20240502 07:59:05.022253 22440 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 849ed56820c54ef892aea88ec91ccc36 (table rebalancer_dynamic_table [id=c1d68ada8f50465e8eca085d5519586b]). Expected version 1 got 0
I20240502 07:59:05.022647 32571 tablet.cc:1722] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:05.022951 32571 tablet.cc:1620] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:59:05.024699 32569 tablet.cc:1722] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:05.024982 32569 tablet.cc:1620] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:59:05.025416 22436 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet 849ed56820c54ef892aea88ec91ccc36 (table rebalancer_dynamic_table [id=c1d68ada8f50465e8eca085d5519586b]). Expected version 1 got 0
I20240502 07:59:05.027165 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=c1d68ada8f50465e8eca085d5519586b] alter complete (version 1)
W20240502 07:59:05.028561 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 07:59:05.040441 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 07:59:05.061148 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:05.083972 32502 raft_consensus.cc:482] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:05.084139 32502 raft_consensus.cc:504] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:05.084610 32502 leader_election.cc:288] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:05.084916 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6d041e9b9503402ba45f5b0dfc777ca5" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:05.084921 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6d041e9b9503402ba45f5b0dfc777ca5" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:05.085042 25266 raft_consensus.cc:2454] T 6d041e9b9503402ba45f5b0dfc777ca5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:05.085047 22717 raft_consensus.cc:2454] T 6d041e9b9503402ba45f5b0dfc777ca5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:05.085300 22521 leader_election.cc:302] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7eced220cf1642f3873fc385b6191e69, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:05.085505 32502 raft_consensus.cc:2781] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:05.085553 32502 raft_consensus.cc:482] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:05.085604 32502 raft_consensus.cc:3037] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.086634 32502 raft_consensus.cc:504] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:05.086882 32502 leader_election.cc:288] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:05.087136 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6d041e9b9503402ba45f5b0dfc777ca5" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:05.087153 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6d041e9b9503402ba45f5b0dfc777ca5" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:05.087229 22717 raft_consensus.cc:3037] T 6d041e9b9503402ba45f5b0dfc777ca5 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.087239 25266 raft_consensus.cc:3037] T 6d041e9b9503402ba45f5b0dfc777ca5 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.087924 22717 raft_consensus.cc:2454] T 6d041e9b9503402ba45f5b0dfc777ca5 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:05.088038 25266 raft_consensus.cc:2454] T 6d041e9b9503402ba45f5b0dfc777ca5 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:05.088181 22519 leader_election.cc:302] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:05.088469 32502 raft_consensus.cc:2781] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:05.088555 32502 raft_consensus.cc:686] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:59:05.088716 32502 consensus_queue.cc:237] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:05.091012 22440 catalog_manager.cc:5484] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } }
W20240502 07:59:05.097183 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:05.131256 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:59:05.136699 32569 tablet.cc:1722] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:05.137045 32569 tablet.cc:1620] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:59:05.137734 32571 tablet.cc:1722] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:05.138181 32571 tablet.cc:1620] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:59:05.138144 32570 tablet.cc:1722] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:05.138408 32570 tablet.cc:1620] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:05.143254 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=c1d68ada8f50465e8eca085d5519586b] alter complete (version 2)
W20240502 07:59:05.170823 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 07:59:05.200500 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 62 similar messages]
W20240502 07:59:05.211503 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 07:59:05.212119 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:05.224367 32199 raft_consensus.cc:482] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:05.224468 32199 raft_consensus.cc:504] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.224844 32199 leader_election.cc:288] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:59:05.225159 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5e734db2608e4a01a20d113c8280b559" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:05.225155 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5e734db2608e4a01a20d113c8280b559" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:05.225291 22584 raft_consensus.cc:2454] T 5e734db2608e4a01a20d113c8280b559 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:05.225291 24872 raft_consensus.cc:2454] T 5e734db2608e4a01a20d113c8280b559 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:05.225540 22781 leader_election.cc:302] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:05.225699 32199 raft_consensus.cc:2781] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:05.225769 32199 raft_consensus.cc:482] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:05.225826 32199 raft_consensus.cc:3037] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.226583 32199 raft_consensus.cc:504] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.226886 32199 leader_election.cc:288] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:59:05.227103 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5e734db2608e4a01a20d113c8280b559" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:05.227208 22584 raft_consensus.cc:3037] T 5e734db2608e4a01a20d113c8280b559 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.227181 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5e734db2608e4a01a20d113c8280b559" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:05.227286 24870 raft_consensus.cc:3037] T 5e734db2608e4a01a20d113c8280b559 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.227957 22584 raft_consensus.cc:2454] T 5e734db2608e4a01a20d113c8280b559 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:05.228046 24870 raft_consensus.cc:2454] T 5e734db2608e4a01a20d113c8280b559 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:05.228237 22784 leader_election.cc:302] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:05.228412 32198 raft_consensus.cc:2781] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:05.228516 32198 raft_consensus.cc:686] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:59:05.228677 32198 consensus_queue.cc:237] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.231876 22444 catalog_manager.cc:5484] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:05.245950 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:59:05.250473 32569 tablet.cc:1722] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:05.250968 32570 tablet.cc:1722] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:05.251299 32571 tablet.cc:1722] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:05.255952 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=c1d68ada8f50465e8eca085d5519586b] alter complete (version 3)
W20240502 07:59:05.305697 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 07:59:05.308230 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20240502 07:59:05.344296 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20240502 07:59:05.358372 22444 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:05.358503 22444 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:05.360450 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 849ed56820c54ef892aea88ec91ccc36
I20240502 07:59:05.361001 25113 tablet_service.cc:1514] Processing DeleteTablet for tablet 849ed56820c54ef892aea88ec91ccc36 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:05 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:59:05.361128 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet 849ed56820c54ef892aea88ec91ccc36 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:05 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:59:05.361258 22695 tablet_service.cc:1514] Processing DeleteTablet for tablet 849ed56820c54ef892aea88ec91ccc36 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:05 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:59:05.361438 32605 tablet_replica.cc:330] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:59:05.361582 32606 tablet_replica.cc:330] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:59:05.361620 32605 raft_consensus.cc:2229] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:05.361745 32605 raft_consensus.cc:2258] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:05.361748 32606 raft_consensus.cc:2229] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:59:05.361817 32607 tablet_replica.cc:330] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:59:05.361992 32607 raft_consensus.cc:2229] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:05.362052 32606 raft_consensus.cc:2258] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:05.362155 32607 raft_consensus.cc:2258] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:05.362941 32607 ts_tablet_manager.cc:1875] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:05.362980 32605 ts_tablet_manager.cc:1875] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:05.362941 32606 ts_tablet_manager.cc:1875] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:05.364909 32605 ts_tablet_manager.cc:1888] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:05.365065 32605 log.cc:1196] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/849ed56820c54ef892aea88ec91ccc36
I20240502 07:59:05.365082 32606 ts_tablet_manager.cc:1888] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:05.365173 32606 log.cc:1196] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/849ed56820c54ef892aea88ec91ccc36
I20240502 07:59:05.365489 32605 ts_tablet_manager.cc:1909] T 849ed56820c54ef892aea88ec91ccc36 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:59:05.365567 32606 ts_tablet_manager.cc:1909] T 849ed56820c54ef892aea88ec91ccc36 P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:59:05.366194 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 849ed56820c54ef892aea88ec91ccc36 (table rebalancer_dynamic_table_new_name [id=c1d68ada8f50465e8eca085d5519586b]) successfully deleted
I20240502 07:59:05.366461 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet 849ed56820c54ef892aea88ec91ccc36 (table rebalancer_dynamic_table_new_name [id=c1d68ada8f50465e8eca085d5519586b]) successfully deleted
I20240502 07:59:05.366787 32607 ts_tablet_manager.cc:1888] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:05.366885 32607 log.cc:1196] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/849ed56820c54ef892aea88ec91ccc36
I20240502 07:59:05.367158 32607 ts_tablet_manager.cc:1909] T 849ed56820c54ef892aea88ec91ccc36 P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:59:05.367807 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet 849ed56820c54ef892aea88ec91ccc36 (table rebalancer_dynamic_table_new_name [id=c1d68ada8f50465e8eca085d5519586b]) successfully deleted
W20240502 07:59:05.369271 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 07:59:05.416127 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20240502 07:59:05.433069 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:05.462030 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:59:05.466871 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 37eef4eca4184134b5672cfe84f6d413 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=23f8dd0878614354a9095ee7df26c25a]), partition=
I20240502 07:59:05.467080 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 37eef4eca4184134b5672cfe84f6d413 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=23f8dd0878614354a9095ee7df26c25a]), partition=
I20240502 07:59:05.467128 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 37eef4eca4184134b5672cfe84f6d413. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:05.467295 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 37eef4eca4184134b5672cfe84f6d413. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:05.468874 32638 tablet_bootstrap.cc:492] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:05.469111 32639 tablet_bootstrap.cc:492] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:05.469141 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 37eef4eca4184134b5672cfe84f6d413 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=23f8dd0878614354a9095ee7df26c25a]), partition=
I20240502 07:59:05.469377 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 37eef4eca4184134b5672cfe84f6d413. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:05.469810 32638 tablet_bootstrap.cc:654] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:05.469818 32639 tablet_bootstrap.cc:654] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:05.470822 32639 tablet_bootstrap.cc:492] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:05.470898 32639 ts_tablet_manager.cc:1367] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:05.471338 32639 raft_consensus.cc:348] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:05.471473 32639 raft_consensus.cc:374] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:05.471506 32639 raft_consensus.cc:729] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:05.471647 32642 tablet_bootstrap.cc:492] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:05.471640 32639 consensus_queue.cc:260] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:05.471925 32639 ts_tablet_manager.cc:1398] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:05.472502 32642 tablet_bootstrap.cc:654] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:05.472537 32638 tablet_bootstrap.cc:492] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:05.472677 32638 ts_tablet_manager.cc:1367] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:59:05.473145 32638 raft_consensus.cc:348] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:05.473278 32638 raft_consensus.cc:374] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:05.473338 32638 raft_consensus.cc:729] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:05.473512 32638 consensus_queue.cc:260] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:05.473834 32638 ts_tablet_manager.cc:1398] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:05.475288 32642 tablet_bootstrap.cc:492] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:05.475423 32642 ts_tablet_manager.cc:1367] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.002s
I20240502 07:59:05.475921 32642 raft_consensus.cc:348] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:05.476048 32642 raft_consensus.cc:374] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:05.476097 32642 raft_consensus.cc:729] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:05.476212 32642 consensus_queue.cc:260] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:05.476521 32642 ts_tablet_manager.cc:1398] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
W20240502 07:59:05.492614 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:05.502252 32565 consensus_queue.cc:1035] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:05.505311 32422 consensus_queue.cc:1035] T 2e55a187ef9b4eabaa3c4a83c1c3b0da P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:05.529366 32502 consensus_queue.cc:1035] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:05.532842 32258 consensus_queue.cc:1035] T 6d041e9b9503402ba45f5b0dfc777ca5 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:05.591936 32520 raft_consensus.cc:482] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:05.592046 32520 raft_consensus.cc:504] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:05.592423 32520 leader_election.cc:288] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:05.592752 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "37eef4eca4184134b5672cfe84f6d413" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:05.592779 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "37eef4eca4184134b5672cfe84f6d413" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:05.592883 24872 raft_consensus.cc:2454] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:59:05.592900 22979 raft_consensus.cc:2454] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:59:05.593127 25200 leader_election.cc:302] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:05.593320 32520 raft_consensus.cc:2781] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:05.593380 32520 raft_consensus.cc:482] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:05.593417 32520 raft_consensus.cc:3037] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.594452 32520 raft_consensus.cc:504] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:05.594810 32520 leader_election.cc:288] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:05.595057 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "37eef4eca4184134b5672cfe84f6d413" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:05.595098 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "37eef4eca4184134b5672cfe84f6d413" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:05.595173 24872 raft_consensus.cc:3037] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.595187 22979 raft_consensus.cc:3037] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.596105 24872 raft_consensus.cc:2454] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:59:05.596103 22979 raft_consensus.cc:2454] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:59:05.596350 25199 leader_election.cc:302] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:05.596510 32520 raft_consensus.cc:2781] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:05.596582 32520 raft_consensus.cc:686] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:59:05.596724 32520 consensus_queue.cc:237] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:05.598793 22444 catalog_manager.cc:5484] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } }
W20240502 07:59:05.615880 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:05.616869 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 07:59:05.716822 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:05.723374 32198 consensus_queue.cc:1035] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:05.724649 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:59:05.729115 24872 raft_consensus.cc:1261] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:05.729578 32521 consensus_queue.cc:1035] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:05.729835 22978 raft_consensus.cc:1261] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:05.730413 32520 consensus_queue.cc:1035] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:05.733867 22444 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet 37eef4eca4184134b5672cfe84f6d413 (table rebalancer_dynamic_table [id=23f8dd0878614354a9095ee7df26c25a]). Expected version 1 got 0
I20240502 07:59:05.736802 22444 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet 37eef4eca4184134b5672cfe84f6d413 (table rebalancer_dynamic_table [id=23f8dd0878614354a9095ee7df26c25a]). Expected version 1 got 0
I20240502 07:59:05.737736 32570 tablet.cc:1722] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:05.738037 32570 tablet.cc:1620] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:05.739872 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_62"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:59:05.743157 32686 tablet.cc:1722] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:05.743429 32686 tablet.cc:1620] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:59:05.745400 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=23f8dd0878614354a9095ee7df26c25a] alter complete (version 1)
I20240502 07:59:05.747197 32198 consensus_queue.cc:1035] T 5e734db2608e4a01a20d113c8280b559 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:05.747859 22444 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet 37eef4eca4184134b5672cfe84f6d413 (table rebalancer_dynamic_table [id=23f8dd0878614354a9095ee7df26c25a]). Expected version 1 got 0
I20240502 07:59:05.749439 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 678b4c744c1c4a4abcb8571bc27f1cf2 (DEFAULT_TABLE table=rebalancer_extra_table_62 [id=a748bd04c57447fbbfd68be262093264]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:05.749723 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 678b4c744c1c4a4abcb8571bc27f1cf2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:05.749996 24850 tablet_service.cc:1467] Processing CreateTablet for tablet 0138df17988d43a89cca165f21b710ca (DEFAULT_TABLE table=rebalancer_extra_table_62 [id=a748bd04c57447fbbfd68be262093264]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:05.750258 24850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0138df17988d43a89cca165f21b710ca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:05.749996 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 66f99f23db244ce3b85035b8dfad9650 (DEFAULT_TABLE table=rebalancer_extra_table_62 [id=a748bd04c57447fbbfd68be262093264]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:05.751158 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 66f99f23db244ce3b85035b8dfad9650. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:05.752146 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 678b4c744c1c4a4abcb8571bc27f1cf2 (DEFAULT_TABLE table=rebalancer_extra_table_62 [id=a748bd04c57447fbbfd68be262093264]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:05.752169 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 678b4c744c1c4a4abcb8571bc27f1cf2 (DEFAULT_TABLE table=rebalancer_extra_table_62 [id=a748bd04c57447fbbfd68be262093264]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:05.752243 32638 tablet_bootstrap.cc:492] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:05.752379 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 678b4c744c1c4a4abcb8571bc27f1cf2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:05.752383 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 678b4c744c1c4a4abcb8571bc27f1cf2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:05.753152 32638 tablet_bootstrap.cc:654] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:05.753485 32690 tablet_bootstrap.cc:492] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:05.754127 32638 tablet_bootstrap.cc:492] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:05.754191 32638 ts_tablet_manager.cc:1367] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:59:05.754518 32690 tablet_bootstrap.cc:654] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:05.754583 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 66f99f23db244ce3b85035b8dfad9650 (DEFAULT_TABLE table=rebalancer_extra_table_62 [id=a748bd04c57447fbbfd68be262093264]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:05.754640 32638 raft_consensus.cc:348] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.754747 32638 raft_consensus.cc:374] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:05.754776 32638 raft_consensus.cc:729] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:05.754807 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 66f99f23db244ce3b85035b8dfad9650. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:05.754899 32638 consensus_queue.cc:260] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.755208 32638 ts_tablet_manager.cc:1398] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:05.755373 32638 tablet_bootstrap.cc:492] T 66f99f23db244ce3b85035b8dfad9650 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:05.755613 32690 tablet_bootstrap.cc:492] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:05.755690 32690 ts_tablet_manager.cc:1367] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:05.756032 22826 tablet_service.cc:1467] Processing CreateTablet for tablet 0138df17988d43a89cca165f21b710ca (DEFAULT_TABLE table=rebalancer_extra_table_62 [id=a748bd04c57447fbbfd68be262093264]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:05.756157 32638 tablet_bootstrap.cc:654] T 66f99f23db244ce3b85035b8dfad9650 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:05.756136 32690 raft_consensus.cc:348] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:05.756222 32690 raft_consensus.cc:374] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:05.756249 32690 raft_consensus.cc:729] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:05.756289 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0138df17988d43a89cca165f21b710ca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:05.756348 32690 consensus_queue.cc:260] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:05.756624 32690 ts_tablet_manager.cc:1398] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:05.757529 32642 tablet_bootstrap.cc:492] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:05.758270 32696 tablet_bootstrap.cc:492] T 0138df17988d43a89cca165f21b710ca P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:05.758283 32642 tablet_bootstrap.cc:654] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:05.758554 22696 tablet_service.cc:1467] Processing CreateTablet for tablet 0138df17988d43a89cca165f21b710ca (DEFAULT_TABLE table=rebalancer_extra_table_62 [id=a748bd04c57447fbbfd68be262093264]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:05.758774 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0138df17988d43a89cca165f21b710ca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:05.759099 32638 tablet_bootstrap.cc:492] T 66f99f23db244ce3b85035b8dfad9650 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:05.759155 32638 ts_tablet_manager.cc:1367] T 66f99f23db244ce3b85035b8dfad9650 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.001s
I20240502 07:59:05.759866 32696 tablet_bootstrap.cc:654] T 0138df17988d43a89cca165f21b710ca P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:05.760569 32690 tablet_bootstrap.cc:492] T 0138df17988d43a89cca165f21b710ca P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:05.760756 32638 raft_consensus.cc:348] T 66f99f23db244ce3b85035b8dfad9650 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.760843 32638 raft_consensus.cc:374] T 66f99f23db244ce3b85035b8dfad9650 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:05.760872 32638 raft_consensus.cc:729] T 66f99f23db244ce3b85035b8dfad9650 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:05.760975 32638 consensus_queue.cc:260] T 66f99f23db244ce3b85035b8dfad9650 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.761122 32696 tablet_bootstrap.cc:492] T 0138df17988d43a89cca165f21b710ca P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:05.761186 32696 ts_tablet_manager.cc:1367] T 0138df17988d43a89cca165f21b710ca P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:59:05.761354 32690 tablet_bootstrap.cc:654] T 0138df17988d43a89cca165f21b710ca P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:05.761514 32638 ts_tablet_manager.cc:1398] T 66f99f23db244ce3b85035b8dfad9650 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:05.749439 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 66f99f23db244ce3b85035b8dfad9650 (DEFAULT_TABLE table=rebalancer_extra_table_62 [id=a748bd04c57447fbbfd68be262093264]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:05.761981 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 66f99f23db244ce3b85035b8dfad9650. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:05.762259 32696 raft_consensus.cc:348] T 0138df17988d43a89cca165f21b710ca P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.762362 32696 raft_consensus.cc:374] T 0138df17988d43a89cca165f21b710ca P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:05.762394 32696 raft_consensus.cc:729] T 0138df17988d43a89cca165f21b710ca P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:05.762492 32696 consensus_queue.cc:260] T 0138df17988d43a89cca165f21b710ca P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.762727 32696 ts_tablet_manager.cc:1398] T 0138df17988d43a89cca165f21b710ca P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:05.763171 32642 tablet_bootstrap.cc:492] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:05.763244 32642 ts_tablet_manager.cc:1367] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.002s
I20240502 07:59:05.763712 32642 raft_consensus.cc:348] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.763804 32642 raft_consensus.cc:374] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:05.763837 32642 raft_consensus.cc:729] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:05.763940 32642 consensus_queue.cc:260] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.764219 32642 ts_tablet_manager.cc:1398] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:05.764560 32690 tablet_bootstrap.cc:492] T 0138df17988d43a89cca165f21b710ca P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:05.764659 32690 ts_tablet_manager.cc:1367] T 0138df17988d43a89cca165f21b710ca P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:59:05.765123 32690 raft_consensus.cc:348] T 0138df17988d43a89cca165f21b710ca P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.765218 32690 raft_consensus.cc:374] T 0138df17988d43a89cca165f21b710ca P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:05.765250 32690 raft_consensus.cc:729] T 0138df17988d43a89cca165f21b710ca P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:05.765362 32690 consensus_queue.cc:260] T 0138df17988d43a89cca165f21b710ca P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.765652 32690 ts_tablet_manager.cc:1398] T 0138df17988d43a89cca165f21b710ca P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:05.765828 32690 tablet_bootstrap.cc:492] T 66f99f23db244ce3b85035b8dfad9650 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:05.766602 32690 tablet_bootstrap.cc:654] T 66f99f23db244ce3b85035b8dfad9650 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:05.768316 32690 tablet_bootstrap.cc:492] T 66f99f23db244ce3b85035b8dfad9650 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:05.768391 32690 ts_tablet_manager.cc:1367] T 66f99f23db244ce3b85035b8dfad9650 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.001s
I20240502 07:59:05.770709 32692 tablet_bootstrap.cc:492] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:05.771868 22444 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet 37eef4eca4184134b5672cfe84f6d413 (table rebalancer_dynamic_table [id=23f8dd0878614354a9095ee7df26c25a]). Expected version 1 got 0
I20240502 07:59:05.772251 32692 tablet_bootstrap.cc:654] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:05.772675 32690 raft_consensus.cc:348] T 66f99f23db244ce3b85035b8dfad9650 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.772816 32690 raft_consensus.cc:374] T 66f99f23db244ce3b85035b8dfad9650 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:05.772891 32690 raft_consensus.cc:729] T 66f99f23db244ce3b85035b8dfad9650 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:05.773078 32690 consensus_queue.cc:260] T 66f99f23db244ce3b85035b8dfad9650 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.773254 32693 tablet_bootstrap.cc:492] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:05.773358 32690 ts_tablet_manager.cc:1398] T 66f99f23db244ce3b85035b8dfad9650 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:59:05.773604 32702 tablet.cc:1722] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:05.773861 32702 tablet.cc:1620] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:59:05.774135 32693 tablet_bootstrap.cc:654] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:05.775259 32692 tablet_bootstrap.cc:492] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:05.775269 32693 tablet_bootstrap.cc:492] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:05.775364 32693 ts_tablet_manager.cc:1367] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:05.775399 32692 ts_tablet_manager.cc:1367] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:59:05.775882 32692 raft_consensus.cc:348] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:05.775985 32692 raft_consensus.cc:374] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:05.776012 32692 raft_consensus.cc:729] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:05.776111 32692 consensus_queue.cc:260] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:05.776444 32692 ts_tablet_manager.cc:1398] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:05.777758 32693 raft_consensus.cc:348] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:05.777887 32693 raft_consensus.cc:374] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:05.777937 32693 raft_consensus.cc:729] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:05.778055 32693 consensus_queue.cc:260] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:05.778676 32693 ts_tablet_manager.cc:1398] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:59:05.796615 32421 raft_consensus.cc:482] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:05.796733 32421 raft_consensus.cc:504] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.797034 32514 raft_consensus.cc:482] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:05.797078 32421 leader_election.cc:288] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:05.797150 32514 raft_consensus.cc:504] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.797360 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0138df17988d43a89cca165f21b710ca" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:05.797364 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0138df17988d43a89cca165f21b710ca" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:05.797468 22717 raft_consensus.cc:2454] T 0138df17988d43a89cca165f21b710ca P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:05.797492 22847 raft_consensus.cc:2454] T 0138df17988d43a89cca165f21b710ca P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:05.797545 32514 leader_election.cc:288] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:59:05.797758 24808 leader_election.cc:302] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:05.797883 32421 raft_consensus.cc:2781] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:05.797940 32421 raft_consensus.cc:482] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:05.797972 32421 raft_consensus.cc:3037] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.798072 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "66f99f23db244ce3b85035b8dfad9650" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:05.798167 24872 raft_consensus.cc:2454] T 66f99f23db244ce3b85035b8dfad9650 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:59:05.798379 22912 leader_election.cc:302] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:05.798511 32258 raft_consensus.cc:482] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:05.798583 32258 raft_consensus.cc:504] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:05.798949 32258 leader_election.cc:288] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:05.798920 32421 raft_consensus.cc:504] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.799141 25133 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "678b4c744c1c4a4abcb8571bc27f1cf2" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:05.799217 32421 leader_election.cc:288] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:05.799253 25133 raft_consensus.cc:2454] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:05.799496 22522 leader_election.cc:302] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:05.799508 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0138df17988d43a89cca165f21b710ca" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:59:05.799508 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0138df17988d43a89cca165f21b710ca" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:05.799594 22847 raft_consensus.cc:3037] T 0138df17988d43a89cca165f21b710ca P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.799604 22716 raft_consensus.cc:3037] T 0138df17988d43a89cca165f21b710ca P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.799737 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "66f99f23db244ce3b85035b8dfad9650" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:05.799829 22717 raft_consensus.cc:2454] T 66f99f23db244ce3b85035b8dfad9650 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:59:05.800441 22716 raft_consensus.cc:2454] T 0138df17988d43a89cca165f21b710ca P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:05.800441 22847 raft_consensus.cc:2454] T 0138df17988d43a89cca165f21b710ca P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:05.800451 32258 raft_consensus.cc:2781] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:05.800504 32258 raft_consensus.cc:482] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:05.800531 32258 raft_consensus.cc:3037] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.800613 32514 raft_consensus.cc:2781] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:05.800678 32514 raft_consensus.cc:482] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:05.800714 32514 raft_consensus.cc:3037] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.801240 32258 raft_consensus.cc:504] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:05.801455 32514 raft_consensus.cc:504] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.801530 32258 leader_election.cc:288] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:05.801753 32514 leader_election.cc:288] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:59:05.801786 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "678b4c744c1c4a4abcb8571bc27f1cf2" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:05.801887 25134 raft_consensus.cc:3037] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.801882 24805 leader_election.cc:302] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:05.802094 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "678b4c744c1c4a4abcb8571bc27f1cf2" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:05.802179 22716 raft_consensus.cc:2454] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:05.802300 32421 raft_consensus.cc:2781] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:05.802484 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "66f99f23db244ce3b85035b8dfad9650" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:05.802557 22716 raft_consensus.cc:3037] T 66f99f23db244ce3b85035b8dfad9650 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.802829 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "678b4c744c1c4a4abcb8571bc27f1cf2" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:05.802884 25134 raft_consensus.cc:2454] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:05.802917 22717 raft_consensus.cc:3037] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.803318 22716 raft_consensus.cc:2454] T 66f99f23db244ce3b85035b8dfad9650 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:59:05.803442 32421 raft_consensus.cc:686] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:59:05.803571 32421 consensus_queue.cc:237] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.803660 22717 raft_consensus.cc:2454] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:05.803915 22522 leader_election.cc:302] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:05.804049 32258 raft_consensus.cc:2781] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:05.804075 22912 leader_election.cc:302] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:05.804134 32258 raft_consensus.cc:686] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:59:05.804255 32258 consensus_queue.cc:237] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:05.804410 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "66f99f23db244ce3b85035b8dfad9650" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:05.804497 24872 raft_consensus.cc:3037] T 66f99f23db244ce3b85035b8dfad9650 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:05.805411 24872 raft_consensus.cc:2454] T 66f99f23db244ce3b85035b8dfad9650 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:59:05.805801 32514 raft_consensus.cc:2781] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:05.805891 32514 raft_consensus.cc:686] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:59:05.806011 32514 consensus_queue.cc:237] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:05.806160 22440 catalog_manager.cc:5484] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } }
I20240502 07:59:05.807660 22436 catalog_manager.cc:5484] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:05.808136 22444 catalog_manager.cc:5484] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
W20240502 07:59:05.810904 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
I20240502 07:59:05.872453 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:59:05.877529 32686 tablet.cc:1722] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:05.877983 32686 tablet.cc:1620] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:59:05.878571 32702 tablet.cc:1722] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:05.878731 32570 tablet.cc:1722] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:05.878850 32702 tablet.cc:1620] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:59:05.878964 32570 tablet.cc:1620] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:05.884119 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=23f8dd0878614354a9095ee7df26c25a] alter complete (version 2)
W20240502 07:59:05.898598 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 07:59:05.936331 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 07:59:05.961038 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:05.987847 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:59:05.992174 32686 tablet.cc:1722] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:05.992911 32702 tablet.cc:1722] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:05.994726 32570 tablet.cc:1722] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:05.997577 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=23f8dd0878614354a9095ee7df26c25a] alter complete (version 3)
W20240502 07:59:06.036460 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 07:59:06.036806 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:06.101824 22440 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:06.102000 22440 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:06.104105 22440 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 37eef4eca4184134b5672cfe84f6d413
I20240502 07:59:06.104619 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet 37eef4eca4184134b5672cfe84f6d413 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:06 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:59:06.104951 32767 tablet_replica.cc:330] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:59:06.105049 25235 tablet_service.cc:1514] Processing DeleteTablet for tablet 37eef4eca4184134b5672cfe84f6d413 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:06 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:59:06.105098 32767 raft_consensus.cc:2229] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:06.105155 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet 37eef4eca4184134b5672cfe84f6d413 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:06 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:59:06.105217 32767 raft_consensus.cc:2258] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:06.105659 301 tablet_replica.cc:330] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:59:06.105854 301 raft_consensus.cc:2229] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:06.105926 300 tablet_replica.cc:330] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:59:06.105999 301 raft_consensus.cc:2258] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:06.106700 301 ts_tablet_manager.cc:1875] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:06.106807 32767 ts_tablet_manager.cc:1875] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:06.106631 300 raft_consensus.cc:2229] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:59:06.107470 300 raft_consensus.cc:2258] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:06.109108 301 ts_tablet_manager.cc:1888] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:06.109174 301 log.cc:1196] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/37eef4eca4184134b5672cfe84f6d413
I20240502 07:59:06.109289 300 ts_tablet_manager.cc:1875] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:06.109589 301 ts_tablet_manager.cc:1909] T 37eef4eca4184134b5672cfe84f6d413 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:59:06.110258 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 37eef4eca4184134b5672cfe84f6d413 (table rebalancer_dynamic_table_new_name [id=23f8dd0878614354a9095ee7df26c25a]) successfully deleted
I20240502 07:59:06.111274 32767 ts_tablet_manager.cc:1888] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:06.111338 32767 log.cc:1196] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/37eef4eca4184134b5672cfe84f6d413
I20240502 07:59:06.111660 32767 ts_tablet_manager.cc:1909] T 37eef4eca4184134b5672cfe84f6d413 P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:59:06.112015 300 ts_tablet_manager.cc:1888] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:06.112088 300 log.cc:1196] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/37eef4eca4184134b5672cfe84f6d413
I20240502 07:59:06.112362 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet 37eef4eca4184134b5672cfe84f6d413 (table rebalancer_dynamic_table_new_name [id=23f8dd0878614354a9095ee7df26c25a]) successfully deleted
I20240502 07:59:06.112394 300 ts_tablet_manager.cc:1909] T 37eef4eca4184134b5672cfe84f6d413 P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:59:06.112974 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet 37eef4eca4184134b5672cfe84f6d413 (table rebalancer_dynamic_table_new_name [id=23f8dd0878614354a9095ee7df26c25a]) successfully deleted
W20240502 07:59:06.178632 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:06.203248 32741 consensus_queue.cc:1035] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:06.208938 22440 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:59:06.210518 32604 consensus_queue.cc:1035] T 678b4c744c1c4a4abcb8571bc27f1cf2 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:06.214519 25113 tablet_service.cc:1467] Processing CreateTablet for tablet b383a2f182664822bc5c4b5e474a1a3d (DEFAULT_TABLE table=rebalancer_dynamic_table [id=060455ba264b42e99755f8989bc67f66]), partition=
I20240502 07:59:06.214795 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b383a2f182664822bc5c4b5e474a1a3d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:06.214919 24852 tablet_service.cc:1467] Processing CreateTablet for tablet b383a2f182664822bc5c4b5e474a1a3d (DEFAULT_TABLE table=rebalancer_dynamic_table [id=060455ba264b42e99755f8989bc67f66]), partition=
I20240502 07:59:06.215148 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b383a2f182664822bc5c4b5e474a1a3d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:06.216737 32692 tablet_bootstrap.cc:492] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:06.217120 32638 tablet_bootstrap.cc:492] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:06.217518 32692 tablet_bootstrap.cc:654] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:06.217934 32638 tablet_bootstrap.cc:654] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:06.218582 22959 tablet_service.cc:1467] Processing CreateTablet for tablet b383a2f182664822bc5c4b5e474a1a3d (DEFAULT_TABLE table=rebalancer_dynamic_table [id=060455ba264b42e99755f8989bc67f66]), partition=
I20240502 07:59:06.218791 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b383a2f182664822bc5c4b5e474a1a3d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:06.219112 32638 tablet_bootstrap.cc:492] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:06.219172 32638 ts_tablet_manager.cc:1367] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:06.219590 32638 raft_consensus.cc:348] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:06.219679 32638 raft_consensus.cc:374] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:06.219707 32638 raft_consensus.cc:729] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:06.219803 32638 consensus_queue.cc:260] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:06.220033 32638 ts_tablet_manager.cc:1398] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:06.220759 32642 tablet_bootstrap.cc:492] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:06.221803 32642 tablet_bootstrap.cc:654] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:06.222204 32692 tablet_bootstrap.cc:492] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:06.222272 32692 ts_tablet_manager.cc:1367] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:59:06.222625 32692 raft_consensus.cc:348] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:06.222702 32692 raft_consensus.cc:374] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:06.222723 32692 raft_consensus.cc:729] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:06.222808 32692 consensus_queue.cc:260] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:06.223047 32692 ts_tablet_manager.cc:1398] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:06.223078 32642 tablet_bootstrap.cc:492] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:06.223155 32642 ts_tablet_manager.cc:1367] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:59:06.223548 32642 raft_consensus.cc:348] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:06.223665 32642 raft_consensus.cc:374] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:06.223697 32642 raft_consensus.cc:729] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:06.223793 32642 consensus_queue.cc:260] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:06.224045 32642 ts_tablet_manager.cc:1398] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
W20240502 07:59:06.236344 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:06.245401 32236 consensus_queue.cc:1035] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:06.248533 32421 consensus_queue.cc:1035] T 0138df17988d43a89cca165f21b710ca P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:06.300560 32685 consensus_queue.cc:1035] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:06.303364 312 raft_consensus.cc:482] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:06.303478 312 raft_consensus.cc:504] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:06.303934 312 leader_election.cc:288] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:06.304245 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b383a2f182664822bc5c4b5e474a1a3d" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:06.304376 24870 raft_consensus.cc:2454] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:06.304616 25068 leader_election.cc:302] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:06.304800 312 raft_consensus.cc:2781] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:06.304838 312 raft_consensus.cc:482] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:06.304872 312 raft_consensus.cc:3037] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:06.304811 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b383a2f182664822bc5c4b5e474a1a3d" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:06.304944 22979 raft_consensus.cc:2454] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:06.305665 32514 consensus_queue.cc:1035] T 66f99f23db244ce3b85035b8dfad9650 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:06.305946 312 raft_consensus.cc:504] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:06.306423 24872 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b383a2f182664822bc5c4b5e474a1a3d" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:06.306517 24872 raft_consensus.cc:3037] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:06.307278 24872 raft_consensus.cc:2454] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:06.307523 25068 leader_election.cc:302] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:06.307668 32468 raft_consensus.cc:2781] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:06.307808 32468 raft_consensus.cc:686] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:59:06.307921 32468 consensus_queue.cc:237] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:06.309113 312 leader_election.cc:288] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:06.309365 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b383a2f182664822bc5c4b5e474a1a3d" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:06.309458 22979 raft_consensus.cc:3037] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:06.310117 22444 catalog_manager.cc:5484] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } }
I20240502 07:59:06.310461 22979 raft_consensus.cc:2454] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:06.316475 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_64"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:59:06.322625 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 1fc8be157dee4287bfcca8fc51a3134f (DEFAULT_TABLE table=rebalancer_extra_table_64 [id=34ebecf8a9e6430cb7cfb81d185936d0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:06.322639 22826 tablet_service.cc:1467] Processing CreateTablet for tablet 1fc8be157dee4287bfcca8fc51a3134f (DEFAULT_TABLE table=rebalancer_extra_table_64 [id=34ebecf8a9e6430cb7cfb81d185936d0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:06.322872 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1fc8be157dee4287bfcca8fc51a3134f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:06.323189 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1fc8be157dee4287bfcca8fc51a3134f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:06.323642 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 1fc8be157dee4287bfcca8fc51a3134f (DEFAULT_TABLE table=rebalancer_extra_table_64 [id=34ebecf8a9e6430cb7cfb81d185936d0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:06.323887 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1fc8be157dee4287bfcca8fc51a3134f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:06.324875 32642 tablet_bootstrap.cc:492] T 1fc8be157dee4287bfcca8fc51a3134f P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:06.325134 22697 tablet_service.cc:1467] Processing CreateTablet for tablet 6fcd5e3f57ec4de791c43c6b5965ef35 (DEFAULT_TABLE table=rebalancer_extra_table_64 [id=34ebecf8a9e6430cb7cfb81d185936d0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:06.325343 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6fcd5e3f57ec4de791c43c6b5965ef35. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:06.325842 32642 tablet_bootstrap.cc:654] T 1fc8be157dee4287bfcca8fc51a3134f P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:06.326115 328 tablet_bootstrap.cc:492] T 1fc8be157dee4287bfcca8fc51a3134f P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:06.327039 327 tablet_bootstrap.cc:492] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:06.327112 25246 tablet_service.cc:1467] Processing CreateTablet for tablet e23ca105bcb14ba39a500d10883dfb59 (DEFAULT_TABLE table=rebalancer_extra_table_64 [id=34ebecf8a9e6430cb7cfb81d185936d0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:06.327240 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 6fcd5e3f57ec4de791c43c6b5965ef35 (DEFAULT_TABLE table=rebalancer_extra_table_64 [id=34ebecf8a9e6430cb7cfb81d185936d0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:06.327345 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e23ca105bcb14ba39a500d10883dfb59. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:06.327455 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6fcd5e3f57ec4de791c43c6b5965ef35. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:06.328150 327 tablet_bootstrap.cc:654] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:06.328717 22826 tablet_service.cc:1467] Processing CreateTablet for tablet e23ca105bcb14ba39a500d10883dfb59 (DEFAULT_TABLE table=rebalancer_extra_table_64 [id=34ebecf8a9e6430cb7cfb81d185936d0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:06.328900 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e23ca105bcb14ba39a500d10883dfb59. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:06.328867 22957 tablet_service.cc:1467] Processing CreateTablet for tablet e23ca105bcb14ba39a500d10883dfb59 (DEFAULT_TABLE table=rebalancer_extra_table_64 [id=34ebecf8a9e6430cb7cfb81d185936d0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:06.329041 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e23ca105bcb14ba39a500d10883dfb59. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:06.329118 327 tablet_bootstrap.cc:492] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:06.329190 327 ts_tablet_manager.cc:1367] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:06.329612 327 raft_consensus.cc:348] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:06.329703 327 raft_consensus.cc:374] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:06.329730 327 raft_consensus.cc:729] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:06.329835 327 consensus_queue.cc:260] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:06.330099 327 ts_tablet_manager.cc:1398] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:06.330420 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 6fcd5e3f57ec4de791c43c6b5965ef35 (DEFAULT_TABLE table=rebalancer_extra_table_64 [id=34ebecf8a9e6430cb7cfb81d185936d0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:06.330670 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6fcd5e3f57ec4de791c43c6b5965ef35. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:06.330770 332 tablet_bootstrap.cc:492] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:06.331570 327 tablet_bootstrap.cc:492] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:06.332353 327 tablet_bootstrap.cc:654] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:06.332846 328 tablet_bootstrap.cc:654] T 1fc8be157dee4287bfcca8fc51a3134f P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:06.333104 332 tablet_bootstrap.cc:654] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:06.333810 327 tablet_bootstrap.cc:492] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:06.333881 327 ts_tablet_manager.cc:1367] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:06.334229 327 raft_consensus.cc:348] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:06.334312 327 raft_consensus.cc:374] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:06.334340 327 raft_consensus.cc:729] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:06.334442 327 consensus_queue.cc:260] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:06.334728 327 ts_tablet_manager.cc:1398] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:06.335767 328 tablet_bootstrap.cc:492] T 1fc8be157dee4287bfcca8fc51a3134f P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:06.336019 328 ts_tablet_manager.cc:1367] T 1fc8be157dee4287bfcca8fc51a3134f P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.000s
I20240502 07:59:06.336378 330 tablet_bootstrap.cc:492] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:06.336445 332 tablet_bootstrap.cc:492] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:06.336575 332 ts_tablet_manager.cc:1367] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:59:06.336557 328 raft_consensus.cc:348] T 1fc8be157dee4287bfcca8fc51a3134f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:06.336740 328 raft_consensus.cc:374] T 1fc8be157dee4287bfcca8fc51a3134f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:06.336808 328 raft_consensus.cc:729] T 1fc8be157dee4287bfcca8fc51a3134f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:06.336980 328 consensus_queue.cc:260] T 1fc8be157dee4287bfcca8fc51a3134f P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:06.337023 332 raft_consensus.cc:348] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:06.337185 332 raft_consensus.cc:374] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:06.337249 332 raft_consensus.cc:729] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:06.337297 330 tablet_bootstrap.cc:654] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:06.337298 328 ts_tablet_manager.cc:1398] T 1fc8be157dee4287bfcca8fc51a3134f P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:06.337445 332 consensus_queue.cc:260] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:06.337801 332 ts_tablet_manager.cc:1398] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:06.337970 332 tablet_bootstrap.cc:492] T e23ca105bcb14ba39a500d10883dfb59 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:06.338601 32642 tablet_bootstrap.cc:492] T 1fc8be157dee4287bfcca8fc51a3134f P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:06.338673 32642 ts_tablet_manager.cc:1367] T 1fc8be157dee4287bfcca8fc51a3134f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.014s user 0.000s sys 0.002s
I20240502 07:59:06.339072 32642 raft_consensus.cc:348] T 1fc8be157dee4287bfcca8fc51a3134f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:06.339165 32642 raft_consensus.cc:374] T 1fc8be157dee4287bfcca8fc51a3134f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:06.339191 32642 raft_consensus.cc:729] T 1fc8be157dee4287bfcca8fc51a3134f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:06.339284 32642 consensus_queue.cc:260] T 1fc8be157dee4287bfcca8fc51a3134f P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:06.339414 332 tablet_bootstrap.cc:654] T e23ca105bcb14ba39a500d10883dfb59 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:06.339561 32642 ts_tablet_manager.cc:1398] T 1fc8be157dee4287bfcca8fc51a3134f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:06.339711 32642 tablet_bootstrap.cc:492] T e23ca105bcb14ba39a500d10883dfb59 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:06.340444 32642 tablet_bootstrap.cc:654] T e23ca105bcb14ba39a500d10883dfb59 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:06.343505 332 tablet_bootstrap.cc:492] T e23ca105bcb14ba39a500d10883dfb59 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:06.343605 32642 tablet_bootstrap.cc:492] T e23ca105bcb14ba39a500d10883dfb59 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:06.343621 332 ts_tablet_manager.cc:1367] T e23ca105bcb14ba39a500d10883dfb59 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:59:06.343669 32642 ts_tablet_manager.cc:1367] T e23ca105bcb14ba39a500d10883dfb59 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:59:06.343986 330 tablet_bootstrap.cc:492] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:06.344041 330 ts_tablet_manager.cc:1367] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.000s
I20240502 07:59:06.344003 332 raft_consensus.cc:348] T e23ca105bcb14ba39a500d10883dfb59 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:06.344003 32642 raft_consensus.cc:348] T e23ca105bcb14ba39a500d10883dfb59 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:06.344082 32642 raft_consensus.cc:374] T e23ca105bcb14ba39a500d10883dfb59 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:06.344082 332 raft_consensus.cc:374] T e23ca105bcb14ba39a500d10883dfb59 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:06.344110 332 raft_consensus.cc:729] T e23ca105bcb14ba39a500d10883dfb59 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:06.344110 32642 raft_consensus.cc:729] T e23ca105bcb14ba39a500d10883dfb59 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:06.344198 332 consensus_queue.cc:260] T e23ca105bcb14ba39a500d10883dfb59 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:06.344208 32642 consensus_queue.cc:260] T e23ca105bcb14ba39a500d10883dfb59 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:06.344414 32642 ts_tablet_manager.cc:1398] T e23ca105bcb14ba39a500d10883dfb59 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:06.344393 330 raft_consensus.cc:348] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:06.344462 330 raft_consensus.cc:374] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:06.344480 330 raft_consensus.cc:729] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:06.344527 32642 tablet_bootstrap.cc:492] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:06.344552 330 consensus_queue.cc:260] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:06.345156 330 ts_tablet_manager.cc:1398] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:06.345374 32642 tablet_bootstrap.cc:654] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:06.345757 332 ts_tablet_manager.cc:1398] T e23ca105bcb14ba39a500d10883dfb59 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:06.346593 32642 tablet_bootstrap.cc:492] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:06.346654 32642 ts_tablet_manager.cc:1367] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:06.346992 32642 raft_consensus.cc:348] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:06.347075 32642 raft_consensus.cc:374] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:06.347100 32642 raft_consensus.cc:729] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:06.348881 32642 consensus_queue.cc:260] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:06.349176 32642 ts_tablet_manager.cc:1398] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
W20240502 07:59:06.381040 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 07:59:06.421396 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:06.424569 32514 raft_consensus.cc:482] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:06.424724 32514 raft_consensus.cc:504] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:06.425072 32514 leader_election.cc:288] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:06.425343 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6fcd5e3f57ec4de791c43c6b5965ef35" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:06.425473 25266 raft_consensus.cc:2454] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:59:06.425455 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6fcd5e3f57ec4de791c43c6b5965ef35" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:06.425582 22717 raft_consensus.cc:2454] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:59:06.425752 22914 leader_election.cc:302] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:06.425936 32685 raft_consensus.cc:2781] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:06.425988 32685 raft_consensus.cc:482] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:06.426028 32685 raft_consensus.cc:3037] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:06.426798 32685 raft_consensus.cc:504] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:06.427073 32685 leader_election.cc:288] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:06.427316 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6fcd5e3f57ec4de791c43c6b5965ef35" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:06.427335 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6fcd5e3f57ec4de791c43c6b5965ef35" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:06.427418 22717 raft_consensus.cc:3037] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:06.427423 25266 raft_consensus.cc:3037] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:06.428176 22717 raft_consensus.cc:2454] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:59:06.428272 25266 raft_consensus.cc:2454] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:59:06.428409 22912 leader_election.cc:302] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:06.428632 32685 raft_consensus.cc:2781] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:06.428774 32685 raft_consensus.cc:686] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:59:06.428889 32685 consensus_queue.cc:237] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:06.431270 22440 catalog_manager.cc:5484] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:06.432176 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:59:06.435333 22978 raft_consensus.cc:1261] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:06.435417 24870 raft_consensus.cc:1261] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:06.435751 312 consensus_queue.cc:1035] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:06.436079 312 consensus_queue.cc:1035] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:06.438632 22444 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet b383a2f182664822bc5c4b5e474a1a3d (table rebalancer_dynamic_table [id=060455ba264b42e99755f8989bc67f66]). Expected version 1 got 0
I20240502 07:59:06.439301 368 tablet.cc:1722] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:06.439587 368 tablet.cc:1620] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:59:06.439591 32570 tablet.cc:1722] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:06.439822 32570 tablet.cc:1620] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:06.441263 22440 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet b383a2f182664822bc5c4b5e474a1a3d (table rebalancer_dynamic_table [id=060455ba264b42e99755f8989bc67f66]). Expected version 1 got 0
I20240502 07:59:06.441639 32702 tablet.cc:1722] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:06.441887 32702 tablet.cc:1620] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:59:06.443029 22444 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet b383a2f182664822bc5c4b5e474a1a3d (table rebalancer_dynamic_table [id=060455ba264b42e99755f8989bc67f66]). Expected version 1 got 0
I20240502 07:59:06.445297 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=060455ba264b42e99755f8989bc67f66] alter complete (version 1)
W20240502 07:59:06.446686 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:06.505383 32298 raft_consensus.cc:482] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:06.505496 32298 raft_consensus.cc:504] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:06.505869 32298 leader_election.cc:288] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:59:06.506132 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1fc8be157dee4287bfcca8fc51a3134f" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:06.506176 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1fc8be157dee4287bfcca8fc51a3134f" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:06.506255 22978 raft_consensus.cc:2454] T 1fc8be157dee4287bfcca8fc51a3134f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:06.506273 22586 raft_consensus.cc:2454] T 1fc8be157dee4287bfcca8fc51a3134f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:06.506477 22782 leader_election.cc:302] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:06.506608 32298 raft_consensus.cc:2781] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:06.506655 32298 raft_consensus.cc:482] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:06.506695 32298 raft_consensus.cc:3037] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:06.507478 32298 raft_consensus.cc:504] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:06.507767 32298 leader_election.cc:288] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:59:06.508023 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1fc8be157dee4287bfcca8fc51a3134f" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:06.508045 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1fc8be157dee4287bfcca8fc51a3134f" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:06.508129 22978 raft_consensus.cc:3037] T 1fc8be157dee4287bfcca8fc51a3134f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:06.508136 22586 raft_consensus.cc:3037] T 1fc8be157dee4287bfcca8fc51a3134f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:06.509083 22978 raft_consensus.cc:2454] T 1fc8be157dee4287bfcca8fc51a3134f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:06.509099 22586 raft_consensus.cc:2454] T 1fc8be157dee4287bfcca8fc51a3134f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:06.509306 22784 leader_election.cc:302] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:06.509526 32298 raft_consensus.cc:2781] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:06.509676 32298 raft_consensus.cc:686] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:59:06.509804 32298 consensus_queue.cc:237] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:06.512097 22444 catalog_manager.cc:5484] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:06.547549 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:59:06.552414 368 tablet.cc:1722] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:06.552788 32570 tablet.cc:1722] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:06.552856 32702 tablet.cc:1722] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:06.553061 32570 tablet.cc:1620] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:06.553083 32702 tablet.cc:1620] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:59:06.555166 368 tablet.cc:1620] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:59:06.558835 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=060455ba264b42e99755f8989bc67f66] alter complete (version 2)
I20240502 07:59:06.597424 32298 raft_consensus.cc:482] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:06.597530 32298 raft_consensus.cc:504] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:06.597882 32298 leader_election.cc:288] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:06.598126 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e23ca105bcb14ba39a500d10883dfb59" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:06.598249 22978 raft_consensus.cc:2454] T e23ca105bcb14ba39a500d10883dfb59 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:06.598331 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e23ca105bcb14ba39a500d10883dfb59" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:06.598448 25266 raft_consensus.cc:2454] T e23ca105bcb14ba39a500d10883dfb59 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:06.598469 22782 leader_election.cc:302] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:06.598654 32298 raft_consensus.cc:2781] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:06.598711 32298 raft_consensus.cc:482] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:06.598745 32298 raft_consensus.cc:3037] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:06.599519 32298 raft_consensus.cc:504] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:06.599802 32298 leader_election.cc:288] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:06.600047 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e23ca105bcb14ba39a500d10883dfb59" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:06.600145 22979 raft_consensus.cc:3037] T e23ca105bcb14ba39a500d10883dfb59 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:06.600425 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e23ca105bcb14ba39a500d10883dfb59" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:06.600533 25266 raft_consensus.cc:3037] T e23ca105bcb14ba39a500d10883dfb59 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:06.600926 22979 raft_consensus.cc:2454] T e23ca105bcb14ba39a500d10883dfb59 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:06.601207 22782 leader_election.cc:302] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:06.601439 32298 raft_consensus.cc:2781] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:06.601506 25266 raft_consensus.cc:2454] T e23ca105bcb14ba39a500d10883dfb59 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:06.601521 32298 raft_consensus.cc:686] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:59:06.601650 32298 consensus_queue.cc:237] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:06.604099 22440 catalog_manager.cc:5484] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
W20240502 07:59:06.610750 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 07:59:06.626965 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20240502 07:59:06.656206 22430 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 3 similar messages]
W20240502 07:59:06.656839 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 5487e66cb4d8416094a1321cb989f018: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:06.662132 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:59:06.667181 368 tablet.cc:1722] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:06.668000 32570 tablet.cc:1722] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:06.672224 32702 tablet.cc:1722] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:06.675520 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=060455ba264b42e99755f8989bc67f66] alter complete (version 3)
I20240502 07:59:06.781301 22440 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:06.781438 22440 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:06.783102 22440 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet b383a2f182664822bc5c4b5e474a1a3d
I20240502 07:59:06.783660 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet b383a2f182664822bc5c4b5e474a1a3d with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:06 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:59:06.783773 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet b383a2f182664822bc5c4b5e474a1a3d with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:06 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:59:06.783914 25113 tablet_service.cc:1514] Processing DeleteTablet for tablet b383a2f182664822bc5c4b5e474a1a3d with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:06 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:59:06.784155 405 tablet_replica.cc:330] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:59:06.784160 404 tablet_replica.cc:330] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:59:06.784287 405 raft_consensus.cc:2229] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:59:06.784287 404 raft_consensus.cc:2229] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:06.784392 404 raft_consensus.cc:2258] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:06.784487 405 raft_consensus.cc:2258] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:06.785041 405 ts_tablet_manager.cc:1875] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:06.785430 403 tablet_replica.cc:330] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:59:06.785553 403 raft_consensus.cc:2229] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:06.785650 403 raft_consensus.cc:2258] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:06.786108 403 ts_tablet_manager.cc:1875] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:06.787261 405 ts_tablet_manager.cc:1888] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:06.787323 405 log.cc:1196] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/b383a2f182664822bc5c4b5e474a1a3d
I20240502 07:59:06.787576 405 ts_tablet_manager.cc:1909] T b383a2f182664822bc5c4b5e474a1a3d P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:59:06.787957 403 ts_tablet_manager.cc:1888] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:06.788024 403 log.cc:1196] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/b383a2f182664822bc5c4b5e474a1a3d
I20240502 07:59:06.788009 404 ts_tablet_manager.cc:1875] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:06.788386 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet b383a2f182664822bc5c4b5e474a1a3d (table rebalancer_dynamic_table_new_name [id=060455ba264b42e99755f8989bc67f66]) successfully deleted
I20240502 07:59:06.788442 403 ts_tablet_manager.cc:1909] T b383a2f182664822bc5c4b5e474a1a3d P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:59:06.788991 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet b383a2f182664822bc5c4b5e474a1a3d (table rebalancer_dynamic_table_new_name [id=060455ba264b42e99755f8989bc67f66]) successfully deleted
I20240502 07:59:06.789685 404 ts_tablet_manager.cc:1888] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:06.789741 404 log.cc:1196] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/b383a2f182664822bc5c4b5e474a1a3d
I20240502 07:59:06.789911 404 ts_tablet_manager.cc:1909] T b383a2f182664822bc5c4b5e474a1a3d P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:59:06.790434 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet b383a2f182664822bc5c4b5e474a1a3d (table rebalancer_dynamic_table_new_name [id=060455ba264b42e99755f8989bc67f66]) successfully deleted
W20240502 07:59:06.800101 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 07:59:06.822739 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:06.865763 32685 consensus_queue.cc:1035] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:06.868170 32685 consensus_queue.cc:1035] T 6fcd5e3f57ec4de791c43c6b5965ef35 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:06.884423 22440 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:59:06.889253 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 12b1b379e2bd4db7b6fc282b32bb432f (DEFAULT_TABLE table=rebalancer_dynamic_table [id=1abca1b91ff74bc689e4e81031ed3ab6]), partition=
I20240502 07:59:06.889430 22826 tablet_service.cc:1467] Processing CreateTablet for tablet 12b1b379e2bd4db7b6fc282b32bb432f (DEFAULT_TABLE table=rebalancer_dynamic_table [id=1abca1b91ff74bc689e4e81031ed3ab6]), partition=
I20240502 07:59:06.889498 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 12b1b379e2bd4db7b6fc282b32bb432f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:06.889614 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 12b1b379e2bd4db7b6fc282b32bb432f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:06.891136 443 tablet_bootstrap.cc:492] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:06.891310 444 tablet_bootstrap.cc:492] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:06.892078 444 tablet_bootstrap.cc:654] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:06.892309 443 tablet_bootstrap.cc:654] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:06.893054 444 tablet_bootstrap.cc:492] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:06.893113 444 ts_tablet_manager.cc:1367] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:06.893275 443 tablet_bootstrap.cc:492] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:06.893275 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 12b1b379e2bd4db7b6fc282b32bb432f (DEFAULT_TABLE table=rebalancer_dynamic_table [id=1abca1b91ff74bc689e4e81031ed3ab6]), partition=
I20240502 07:59:06.893349 443 ts_tablet_manager.cc:1367] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:06.893481 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 12b1b379e2bd4db7b6fc282b32bb432f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:06.893441 444 raft_consensus.cc:348] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:06.893527 444 raft_consensus.cc:374] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:06.893549 444 raft_consensus.cc:729] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:06.893646 444 consensus_queue.cc:260] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:06.893705 443 raft_consensus.cc:348] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:06.893790 443 raft_consensus.cc:374] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:06.893813 443 raft_consensus.cc:729] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:06.893901 443 consensus_queue.cc:260] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:06.894104 443 ts_tablet_manager.cc:1398] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:06.894786 444 ts_tablet_manager.cc:1398] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:06.895511 447 tablet_bootstrap.cc:492] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:06.896404 447 tablet_bootstrap.cc:654] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:06.897580 447 tablet_bootstrap.cc:492] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:06.897681 447 ts_tablet_manager.cc:1367] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:06.898123 447 raft_consensus.cc:348] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:06.898262 447 raft_consensus.cc:374] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:06.898332 447 raft_consensus.cc:729] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:06.898460 447 consensus_queue.cc:260] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:06.898720 447 ts_tablet_manager.cc:1398] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:07.003465 312 raft_consensus.cc:482] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:07.003592 312 raft_consensus.cc:504] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:07.003988 312 leader_election.cc:288] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:07.004237 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "12b1b379e2bd4db7b6fc282b32bb432f" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:07.004271 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "12b1b379e2bd4db7b6fc282b32bb432f" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:07.004380 22979 raft_consensus.cc:2454] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:07.004384 22845 raft_consensus.cc:2454] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:07.004606 25069 leader_election.cc:302] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 4654231dfe994c1a9d2e37d9c3d75699; no voters:
I20240502 07:59:07.004803 312 raft_consensus.cc:2781] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:07.004863 312 raft_consensus.cc:482] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:07.004899 312 raft_consensus.cc:3037] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.005661 312 raft_consensus.cc:504] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:07.005976 312 leader_election.cc:288] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:07.006210 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "12b1b379e2bd4db7b6fc282b32bb432f" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:07.006263 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "12b1b379e2bd4db7b6fc282b32bb432f" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:59:07.006330 22979 raft_consensus.cc:3037] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.006351 22845 raft_consensus.cc:3037] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.007218 22845 raft_consensus.cc:2454] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:07.007218 22979 raft_consensus.cc:2454] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:07.007481 25069 leader_election.cc:302] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 4654231dfe994c1a9d2e37d9c3d75699; no voters:
I20240502 07:59:07.007687 312 raft_consensus.cc:2781] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:07.007750 312 raft_consensus.cc:686] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:59:07.007872 312 consensus_queue.cc:237] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:07.010123 22440 catalog_manager.cc:5484] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } }
I20240502 07:59:07.051704 32298 consensus_queue.cc:1035] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:07.055598 32198 consensus_queue.cc:1035] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:07.057960 32298 consensus_queue.cc:1035] T 1fc8be157dee4287bfcca8fc51a3134f P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:07.062979 32198 consensus_queue.cc:1035] T e23ca105bcb14ba39a500d10883dfb59 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:07.119457 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:07.122395 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:59:07.123293 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_66"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:59:07.126864 22979 raft_consensus.cc:1261] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:07.127235 312 consensus_queue.cc:1035] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:07.127668 22845 raft_consensus.cc:1261] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:07.128018 312 consensus_queue.cc:1035] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:07.129612 22959 tablet_service.cc:1467] Processing CreateTablet for tablet f09319be8bb445839415e70203a53ac1 (DEFAULT_TABLE table=rebalancer_extra_table_66 [id=65d8d9fc065e4ceea25908ba2cf00a90]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.129611 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 3d97fb0fa4c74aab935c021aa317fc79 (DEFAULT_TABLE table=rebalancer_extra_table_66 [id=65d8d9fc065e4ceea25908ba2cf00a90]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.129877 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f09319be8bb445839415e70203a53ac1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.129901 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3d97fb0fa4c74aab935c021aa317fc79. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.130496 22826 tablet_service.cc:1467] Processing CreateTablet for tablet 3d97fb0fa4c74aab935c021aa317fc79 (DEFAULT_TABLE table=rebalancer_extra_table_66 [id=65d8d9fc065e4ceea25908ba2cf00a90]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.130652 22564 tablet_service.cc:1467] Processing CreateTablet for tablet f09319be8bb445839415e70203a53ac1 (DEFAULT_TABLE table=rebalancer_extra_table_66 [id=65d8d9fc065e4ceea25908ba2cf00a90]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.130771 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3d97fb0fa4c74aab935c021aa317fc79. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.130929 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f09319be8bb445839415e70203a53ac1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.132535 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 3d97fb0fa4c74aab935c021aa317fc79 (DEFAULT_TABLE table=rebalancer_extra_table_66 [id=65d8d9fc065e4ceea25908ba2cf00a90]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.132789 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3d97fb0fa4c74aab935c021aa317fc79. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.132872 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 0b4755ea5ee34f25bd258042a28a42e9 (DEFAULT_TABLE table=rebalancer_extra_table_66 [id=65d8d9fc065e4ceea25908ba2cf00a90]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.132999 444 tablet_bootstrap.cc:492] T f09319be8bb445839415e70203a53ac1 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:07.133061 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0b4755ea5ee34f25bd258042a28a42e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.133829 444 tablet_bootstrap.cc:654] T f09319be8bb445839415e70203a53ac1 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.133934 491 tablet_bootstrap.cc:492] T 3d97fb0fa4c74aab935c021aa317fc79 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:07.134903 447 tablet_bootstrap.cc:492] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:07.134948 25115 tablet_service.cc:1467] Processing CreateTablet for tablet f09319be8bb445839415e70203a53ac1 (DEFAULT_TABLE table=rebalancer_extra_table_66 [id=65d8d9fc065e4ceea25908ba2cf00a90]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.135133 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f09319be8bb445839415e70203a53ac1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.135779 444 tablet_bootstrap.cc:492] T f09319be8bb445839415e70203a53ac1 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:07.135800 447 tablet_bootstrap.cc:654] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.135856 444 ts_tablet_manager.cc:1367] T f09319be8bb445839415e70203a53ac1 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:59:07.136339 444 raft_consensus.cc:348] T f09319be8bb445839415e70203a53ac1 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:07.136451 444 raft_consensus.cc:374] T f09319be8bb445839415e70203a53ac1 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.136478 444 raft_consensus.cc:729] T f09319be8bb445839415e70203a53ac1 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.136590 444 consensus_queue.cc:260] T f09319be8bb445839415e70203a53ac1 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:07.136831 491 tablet_bootstrap.cc:654] T 3d97fb0fa4c74aab935c021aa317fc79 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.136886 444 ts_tablet_manager.cc:1398] T f09319be8bb445839415e70203a53ac1 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:07.137826 22436 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet 12b1b379e2bd4db7b6fc282b32bb432f (table rebalancer_dynamic_table [id=1abca1b91ff74bc689e4e81031ed3ab6]). Expected version 1 got 0
I20240502 07:59:07.137933 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 0b4755ea5ee34f25bd258042a28a42e9 (DEFAULT_TABLE table=rebalancer_extra_table_66 [id=65d8d9fc065e4ceea25908ba2cf00a90]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.138187 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0b4755ea5ee34f25bd258042a28a42e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.139032 443 tablet_bootstrap.cc:492] T 3d97fb0fa4c74aab935c021aa317fc79 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:07.139977 443 tablet_bootstrap.cc:654] T 3d97fb0fa4c74aab935c021aa317fc79 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.140441 311 mvcc.cc:205] Tried to move back new op lower bound from 7023152116228108288 to 7023152115745280000. Current Snapshot: MvccSnapshot[applied={T|T < 7023152116228108288}]
I20240502 07:59:07.140514 368 tablet.cc:1722] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:07.140774 368 tablet.cc:1620] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:59:07.141639 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 0b4755ea5ee34f25bd258042a28a42e9 (DEFAULT_TABLE table=rebalancer_extra_table_66 [id=65d8d9fc065e4ceea25908ba2cf00a90]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.141845 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0b4755ea5ee34f25bd258042a28a42e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.142954 22436 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet 12b1b379e2bd4db7b6fc282b32bb432f (table rebalancer_dynamic_table [id=1abca1b91ff74bc689e4e81031ed3ab6]). Expected version 1 got 0
I20240502 07:59:07.146248 447 tablet_bootstrap.cc:492] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:07.146279 443 tablet_bootstrap.cc:492] T 3d97fb0fa4c74aab935c021aa317fc79 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:07.146353 447 ts_tablet_manager.cc:1367] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.000s
I20240502 07:59:07.146358 443 ts_tablet_manager.cc:1367] T 3d97fb0fa4c74aab935c021aa317fc79 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
I20240502 07:59:07.146400 22438 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet 12b1b379e2bd4db7b6fc282b32bb432f (table rebalancer_dynamic_table [id=1abca1b91ff74bc689e4e81031ed3ab6]). Expected version 1 got 0
I20240502 07:59:07.146751 447 raft_consensus.cc:348] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:07.146751 443 raft_consensus.cc:348] T 3d97fb0fa4c74aab935c021aa317fc79 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:07.146845 447 raft_consensus.cc:374] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.146867 443 raft_consensus.cc:374] T 3d97fb0fa4c74aab935c021aa317fc79 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.146876 447 raft_consensus.cc:729] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.146898 443 raft_consensus.cc:729] T 3d97fb0fa4c74aab935c021aa317fc79 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.146991 443 consensus_queue.cc:260] T 3d97fb0fa4c74aab935c021aa317fc79 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:07.146987 447 consensus_queue.cc:260] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:07.147226 443 ts_tablet_manager.cc:1398] T 3d97fb0fa4c74aab935c021aa317fc79 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:07.147226 447 ts_tablet_manager.cc:1398] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:07.147389 447 tablet_bootstrap.cc:492] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:07.148048 22436 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet 12b1b379e2bd4db7b6fc282b32bb432f (table rebalancer_dynamic_table [id=1abca1b91ff74bc689e4e81031ed3ab6]). Expected version 1 got 0
I20240502 07:59:07.148054 493 tablet_bootstrap.cc:492] T 0b4755ea5ee34f25bd258042a28a42e9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:07.148226 447 tablet_bootstrap.cc:654] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.148336 495 tablet_bootstrap.cc:492] T 0b4755ea5ee34f25bd258042a28a42e9 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:07.149587 497 tablet.cc:1722] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:07.151481 447 tablet_bootstrap.cc:492] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:07.151563 447 ts_tablet_manager.cc:1367] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.001s
I20240502 07:59:07.151971 447 raft_consensus.cc:348] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:07.152058 447 raft_consensus.cc:374] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.152011 32570 tablet.cc:1722] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:07.152089 447 raft_consensus.cc:729] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.152180 447 consensus_queue.cc:260] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:07.152318 32570 tablet.cc:1620] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:07.152443 447 ts_tablet_manager.cc:1398] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:07.152565 447 tablet_bootstrap.cc:492] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:07.152807 495 tablet_bootstrap.cc:654] T 0b4755ea5ee34f25bd258042a28a42e9 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.153034 497 tablet.cc:1620] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:59:07.153326 493 tablet_bootstrap.cc:654] T 0b4755ea5ee34f25bd258042a28a42e9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.153342 447 tablet_bootstrap.cc:654] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.155547 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=1abca1b91ff74bc689e4e81031ed3ab6] alter complete (version 1)
I20240502 07:59:07.156072 495 tablet_bootstrap.cc:492] T 0b4755ea5ee34f25bd258042a28a42e9 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:07.156275 495 ts_tablet_manager.cc:1367] T 0b4755ea5ee34f25bd258042a28a42e9 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.000s
I20240502 07:59:07.156571 491 tablet_bootstrap.cc:492] T 3d97fb0fa4c74aab935c021aa317fc79 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:07.156723 495 raft_consensus.cc:348] T 0b4755ea5ee34f25bd258042a28a42e9 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:07.156785 491 ts_tablet_manager.cc:1367] T 3d97fb0fa4c74aab935c021aa317fc79 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.023s user 0.002s sys 0.000s
I20240502 07:59:07.156886 495 raft_consensus.cc:374] T 0b4755ea5ee34f25bd258042a28a42e9 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.156936 495 raft_consensus.cc:729] T 0b4755ea5ee34f25bd258042a28a42e9 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.157068 495 consensus_queue.cc:260] T 0b4755ea5ee34f25bd258042a28a42e9 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:07.157373 495 ts_tablet_manager.cc:1398] T 0b4755ea5ee34f25bd258042a28a42e9 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:07.157541 491 raft_consensus.cc:348] T 3d97fb0fa4c74aab935c021aa317fc79 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:07.157649 491 raft_consensus.cc:374] T 3d97fb0fa4c74aab935c021aa317fc79 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.157703 491 raft_consensus.cc:729] T 3d97fb0fa4c74aab935c021aa317fc79 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.157807 491 consensus_queue.cc:260] T 3d97fb0fa4c74aab935c021aa317fc79 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:07.158116 491 ts_tablet_manager.cc:1398] T 3d97fb0fa4c74aab935c021aa317fc79 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:07.165513 447 tablet_bootstrap.cc:492] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:07.165614 447 ts_tablet_manager.cc:1367] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.013s user 0.000s sys 0.002s
I20240502 07:59:07.165947 493 tablet_bootstrap.cc:492] T 0b4755ea5ee34f25bd258042a28a42e9 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:07.166067 493 ts_tablet_manager.cc:1367] T 0b4755ea5ee34f25bd258042a28a42e9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.018s user 0.002s sys 0.000s
I20240502 07:59:07.166072 447 raft_consensus.cc:348] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:07.166182 447 raft_consensus.cc:374] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.166231 447 raft_consensus.cc:729] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.166355 447 consensus_queue.cc:260] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:07.166607 447 ts_tablet_manager.cc:1398] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:07.166642 493 raft_consensus.cc:348] T 0b4755ea5ee34f25bd258042a28a42e9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:07.166765 493 raft_consensus.cc:374] T 0b4755ea5ee34f25bd258042a28a42e9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.166817 493 raft_consensus.cc:729] T 0b4755ea5ee34f25bd258042a28a42e9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.166944 493 consensus_queue.cc:260] T 0b4755ea5ee34f25bd258042a28a42e9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:07.167196 493 ts_tablet_manager.cc:1398] T 0b4755ea5ee34f25bd258042a28a42e9 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:07.167619 493 tablet_bootstrap.cc:492] T f09319be8bb445839415e70203a53ac1 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:07.168542 493 tablet_bootstrap.cc:654] T f09319be8bb445839415e70203a53ac1 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.171104 493 tablet_bootstrap.cc:492] T f09319be8bb445839415e70203a53ac1 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:07.171196 493 ts_tablet_manager.cc:1367] T f09319be8bb445839415e70203a53ac1 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:59:07.171612 493 raft_consensus.cc:348] T f09319be8bb445839415e70203a53ac1 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:07.171744 493 raft_consensus.cc:374] T f09319be8bb445839415e70203a53ac1 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.171898 493 raft_consensus.cc:729] T f09319be8bb445839415e70203a53ac1 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.172011 493 consensus_queue.cc:260] T f09319be8bb445839415e70203a53ac1 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:07.172322 493 ts_tablet_manager.cc:1398] T f09319be8bb445839415e70203a53ac1 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:07.217857 312 raft_consensus.cc:482] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:07.217984 312 raft_consensus.cc:504] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:07.218336 312 leader_election.cc:288] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:07.220402 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3d97fb0fa4c74aab935c021aa317fc79" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:07.220541 24871 raft_consensus.cc:2454] T 3d97fb0fa4c74aab935c021aa317fc79 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:07.221076 25068 leader_election.cc:302] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:07.221244 312 raft_consensus.cc:2781] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:07.221307 312 raft_consensus.cc:482] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:07.221347 312 raft_consensus.cc:3037] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.221318 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3d97fb0fa4c74aab935c021aa317fc79" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:07.221465 22845 raft_consensus.cc:2454] T 3d97fb0fa4c74aab935c021aa317fc79 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:07.222240 312 raft_consensus.cc:504] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:07.222503 312 leader_election.cc:288] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:07.222746 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3d97fb0fa4c74aab935c021aa317fc79" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:07.222770 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3d97fb0fa4c74aab935c021aa317fc79" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:59:07.222839 24870 raft_consensus.cc:3037] T 3d97fb0fa4c74aab935c021aa317fc79 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.222851 22845 raft_consensus.cc:3037] T 3d97fb0fa4c74aab935c021aa317fc79 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.223579 22845 raft_consensus.cc:2454] T 3d97fb0fa4c74aab935c021aa317fc79 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:07.223618 24870 raft_consensus.cc:2454] T 3d97fb0fa4c74aab935c021aa317fc79 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:07.223822 25071 leader_election.cc:302] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:07.223985 312 raft_consensus.cc:2781] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:07.224063 312 raft_consensus.cc:686] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:59:07.224186 312 consensus_queue.cc:237] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:07.226190 22440 catalog_manager.cc:5484] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:07.261030 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:59:07.265904 312 raft_consensus.cc:482] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:07.266019 312 raft_consensus.cc:504] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:07.266342 368 tablet.cc:1722] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:07.266649 368 tablet.cc:1620] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:59:07.266945 497 tablet.cc:1722] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:07.266996 32570 tablet.cc:1722] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:07.267186 497 tablet.cc:1620] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:59:07.267205 312 leader_election.cc:288] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:07.267231 32570 tablet.cc:1620] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:07.271456 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0b4755ea5ee34f25bd258042a28a42e9" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:07.271579 22584 raft_consensus.cc:2454] T 0b4755ea5ee34f25bd258042a28a42e9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
W20240502 07:59:07.272192 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:07.272243 25071 leader_election.cc:302] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:07.272861 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0b4755ea5ee34f25bd258042a28a42e9" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:07.272979 25266 raft_consensus.cc:2454] T 0b4755ea5ee34f25bd258042a28a42e9 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:07.273769 312 raft_consensus.cc:2781] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:07.273972 312 raft_consensus.cc:482] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:07.274013 312 raft_consensus.cc:3037] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.272846 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=1abca1b91ff74bc689e4e81031ed3ab6] alter complete (version 2)
W20240502 07:59:07.274848 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:07.274947 312 raft_consensus.cc:504] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:07.275264 312 leader_election.cc:288] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:07.275612 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0b4755ea5ee34f25bd258042a28a42e9" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:07.275713 25266 raft_consensus.cc:3037] T 0b4755ea5ee34f25bd258042a28a42e9 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.275705 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0b4755ea5ee34f25bd258042a28a42e9" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:07.275812 22584 raft_consensus.cc:3037] T 0b4755ea5ee34f25bd258042a28a42e9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.276402 25266 raft_consensus.cc:2454] T 0b4755ea5ee34f25bd258042a28a42e9 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:07.276513 22584 raft_consensus.cc:2454] T 0b4755ea5ee34f25bd258042a28a42e9 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:07.276788 25071 leader_election.cc:302] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:07.276917 312 raft_consensus.cc:2781] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:07.277002 312 raft_consensus.cc:686] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:59:07.277119 312 consensus_queue.cc:237] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:07.278972 22444 catalog_manager.cc:5484] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } }
I20240502 07:59:07.355700 505 raft_consensus.cc:482] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:07.355829 505 raft_consensus.cc:504] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:07.356202 505 leader_election.cc:288] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:59:07.356464 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f09319be8bb445839415e70203a53ac1" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:07.356505 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f09319be8bb445839415e70203a53ac1" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:07.356622 22978 raft_consensus.cc:2454] T f09319be8bb445839415e70203a53ac1 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:07.356643 22584 raft_consensus.cc:2454] T f09319be8bb445839415e70203a53ac1 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:07.356905 25069 leader_election.cc:302] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 4654231dfe994c1a9d2e37d9c3d75699; no voters:
I20240502 07:59:07.357093 505 raft_consensus.cc:2781] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:07.357152 505 raft_consensus.cc:482] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:07.357193 505 raft_consensus.cc:3037] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.358105 505 raft_consensus.cc:504] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:07.358425 505 leader_election.cc:288] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:59:07.358718 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f09319be8bb445839415e70203a53ac1" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:07.358750 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f09319be8bb445839415e70203a53ac1" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:07.358824 22584 raft_consensus.cc:3037] T f09319be8bb445839415e70203a53ac1 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.358841 22978 raft_consensus.cc:3037] T f09319be8bb445839415e70203a53ac1 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.359504 22584 raft_consensus.cc:2454] T f09319be8bb445839415e70203a53ac1 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:07.359586 22978 raft_consensus.cc:2454] T f09319be8bb445839415e70203a53ac1 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:07.359776 25071 leader_election.cc:302] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:07.359922 505 raft_consensus.cc:2781] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:07.360005 505 raft_consensus.cc:686] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:59:07.360144 505 consensus_queue.cc:237] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:07.362489 22440 catalog_manager.cc:5484] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
W20240502 07:59:07.375629 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:07.377503 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:59:07.382898 368 tablet.cc:1722] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:07.385121 32570 tablet.cc:1722] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:07.385737 497 tablet.cc:1722] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:07.390081 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=1abca1b91ff74bc689e4e81031ed3ab6] alter complete (version 3)
W20240502 07:59:07.422797 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:07.493026 22436 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:07.493182 22436 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:07.494956 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 12b1b379e2bd4db7b6fc282b32bb432f
I20240502 07:59:07.495481 22959 tablet_service.cc:1514] Processing DeleteTablet for tablet 12b1b379e2bd4db7b6fc282b32bb432f with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:07 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:59:07.495518 22826 tablet_service.cc:1514] Processing DeleteTablet for tablet 12b1b379e2bd4db7b6fc282b32bb432f with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:07 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:59:07.495831 541 tablet_replica.cc:330] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:59:07.496031 25113 tablet_service.cc:1514] Processing DeleteTablet for tablet 12b1b379e2bd4db7b6fc282b32bb432f with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:07 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:59:07.496079 541 raft_consensus.cc:2229] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:07.496096 540 tablet_replica.cc:330] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:59:07.496225 540 raft_consensus.cc:2229] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:07.496206 541 raft_consensus.cc:2258] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:07.496371 540 raft_consensus.cc:2258] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:07.496444 542 tablet_replica.cc:330] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:59:07.496626 542 raft_consensus.cc:2229] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:59:07.496861 542 raft_consensus.cc:2258] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:07.496922 541 ts_tablet_manager.cc:1875] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:07.497395 542 ts_tablet_manager.cc:1875] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:07.497442 540 ts_tablet_manager.cc:1875] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:07.499140 541 ts_tablet_manager.cc:1888] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:07.499217 541 log.cc:1196] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/12b1b379e2bd4db7b6fc282b32bb432f
I20240502 07:59:07.499269 540 ts_tablet_manager.cc:1888] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:07.499349 540 log.cc:1196] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/12b1b379e2bd4db7b6fc282b32bb432f
I20240502 07:59:07.499401 542 ts_tablet_manager.cc:1888] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:07.499497 542 log.cc:1196] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/12b1b379e2bd4db7b6fc282b32bb432f
I20240502 07:59:07.499559 541 ts_tablet_manager.cc:1909] T 12b1b379e2bd4db7b6fc282b32bb432f P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:59:07.499703 540 ts_tablet_manager.cc:1909] T 12b1b379e2bd4db7b6fc282b32bb432f P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:59:07.499756 542 ts_tablet_manager.cc:1909] T 12b1b379e2bd4db7b6fc282b32bb432f P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:59:07.500202 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 12b1b379e2bd4db7b6fc282b32bb432f (table rebalancer_dynamic_table_new_name [id=1abca1b91ff74bc689e4e81031ed3ab6]) successfully deleted
I20240502 07:59:07.500253 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 12b1b379e2bd4db7b6fc282b32bb432f (table rebalancer_dynamic_table_new_name [id=1abca1b91ff74bc689e4e81031ed3ab6]) successfully deleted
I20240502 07:59:07.500358 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet 12b1b379e2bd4db7b6fc282b32bb432f (table rebalancer_dynamic_table_new_name [id=1abca1b91ff74bc689e4e81031ed3ab6]) successfully deleted
W20240502 07:59:07.524842 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 07:59:07.525650 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:07.597918 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:59:07.605055 24852 tablet_service.cc:1467] Processing CreateTablet for tablet bd57ea57711f4562b514353828fb934e (DEFAULT_TABLE table=rebalancer_dynamic_table [id=53c5a7fa93ae4c6fb04483f33b7bcaae]), partition=
I20240502 07:59:07.605306 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd57ea57711f4562b514353828fb934e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.605356 22826 tablet_service.cc:1467] Processing CreateTablet for tablet bd57ea57711f4562b514353828fb934e (DEFAULT_TABLE table=rebalancer_dynamic_table [id=53c5a7fa93ae4c6fb04483f33b7bcaae]), partition=
I20240502 07:59:07.605684 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd57ea57711f4562b514353828fb934e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.606961 491 tablet_bootstrap.cc:492] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:07.607153 25246 tablet_service.cc:1467] Processing CreateTablet for tablet bd57ea57711f4562b514353828fb934e (DEFAULT_TABLE table=rebalancer_dynamic_table [id=53c5a7fa93ae4c6fb04483f33b7bcaae]), partition=
I20240502 07:59:07.607365 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd57ea57711f4562b514353828fb934e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.607838 491 tablet_bootstrap.cc:654] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.609465 495 tablet_bootstrap.cc:492] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:07.610025 443 tablet_bootstrap.cc:492] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:07.610584 495 tablet_bootstrap.cc:654] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.611178 443 tablet_bootstrap.cc:654] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.615466 443 tablet_bootstrap.cc:492] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:07.615594 443 ts_tablet_manager.cc:1367] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:59:07.616367 443 raft_consensus.cc:348] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:07.616549 443 raft_consensus.cc:374] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.616590 443 raft_consensus.cc:729] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.616744 443 consensus_queue.cc:260] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:07.617108 443 ts_tablet_manager.cc:1398] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 07:59:07.617906 495 tablet_bootstrap.cc:492] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:07.617983 495 ts_tablet_manager.cc:1367] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.009s user 0.002s sys 0.000s
I20240502 07:59:07.618373 495 raft_consensus.cc:348] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:07.618479 495 raft_consensus.cc:374] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.618515 495 raft_consensus.cc:729] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.618633 495 consensus_queue.cc:260] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:07.618911 495 ts_tablet_manager.cc:1398] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:07.618990 491 tablet_bootstrap.cc:492] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:07.619060 491 ts_tablet_manager.cc:1367] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.012s user 0.002s sys 0.000s
I20240502 07:59:07.619467 491 raft_consensus.cc:348] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:07.619542 491 raft_consensus.cc:374] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.619562 491 raft_consensus.cc:729] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.619653 491 consensus_queue.cc:260] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:07.619875 491 ts_tablet_manager.cc:1398] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
W20240502 07:59:07.624068 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 07:59:07.656286 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 07:59:07.658982 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:07.697582 505 consensus_queue.cc:1035] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:07.700239 32468 consensus_queue.cc:1035] T 3d97fb0fa4c74aab935c021aa317fc79 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:07.794807 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:07.806560 32236 raft_consensus.cc:482] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:07.806669 32236 raft_consensus.cc:504] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:07.807008 32236 leader_election.cc:288] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:07.807276 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd57ea57711f4562b514353828fb934e" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:07.807308 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd57ea57711f4562b514353828fb934e" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:07.807405 25266 raft_consensus.cc:2454] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:07.807435 22847 raft_consensus.cc:2454] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:07.807739 24807 leader_election.cc:302] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:07.807891 32236 raft_consensus.cc:2781] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:07.807950 32236 raft_consensus.cc:482] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:07.807992 32236 raft_consensus.cc:3037] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.809082 32236 raft_consensus.cc:504] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:07.809386 32236 leader_election.cc:288] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:07.809599 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd57ea57711f4562b514353828fb934e" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:59:07.809703 22845 raft_consensus.cc:3037] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.809725 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bd57ea57711f4562b514353828fb934e" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:07.809809 25266 raft_consensus.cc:3037] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.810612 22845 raft_consensus.cc:2454] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:07.810686 25266 raft_consensus.cc:2454] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:07.810848 24808 leader_election.cc:302] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:07.810964 32236 raft_consensus.cc:2781] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:07.811021 32236 raft_consensus.cc:686] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:59:07.811142 32236 consensus_queue.cc:237] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:07.815425 22440 catalog_manager.cc:5484] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:07.818255 320 consensus_queue.cc:1035] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:07.821527 320 consensus_queue.cc:1035] T 0b4755ea5ee34f25bd258042a28a42e9 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:07.829435 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:07.842736 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 07:59:07.847681 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:07.875980 22440 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_68"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:59:07.881614 25113 tablet_service.cc:1467] Processing CreateTablet for tablet b2d4cffdd0f14378804035004ab246b9 (DEFAULT_TABLE table=rebalancer_extra_table_68 [id=791dd09f806e4b5ea8185c86741b9731]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.881923 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b2d4cffdd0f14378804035004ab246b9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.882504 25115 tablet_service.cc:1467] Processing CreateTablet for tablet a48b2fd96ed448fa974e02b9110e2c49 (DEFAULT_TABLE table=rebalancer_extra_table_68 [id=791dd09f806e4b5ea8185c86741b9731]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.882700 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a48b2fd96ed448fa974e02b9110e2c49. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.882982 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 9a3ed793ccf54fc3ba42f9f13db2aa9f (DEFAULT_TABLE table=rebalancer_extra_table_68 [id=791dd09f806e4b5ea8185c86741b9731]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.883221 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9a3ed793ccf54fc3ba42f9f13db2aa9f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.884716 618 tablet_bootstrap.cc:492] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:07.884850 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 9a3ed793ccf54fc3ba42f9f13db2aa9f (DEFAULT_TABLE table=rebalancer_extra_table_68 [id=791dd09f806e4b5ea8185c86741b9731]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.885083 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9a3ed793ccf54fc3ba42f9f13db2aa9f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.885425 25246 tablet_service.cc:1467] Processing CreateTablet for tablet a48b2fd96ed448fa974e02b9110e2c49 (DEFAULT_TABLE table=rebalancer_extra_table_68 [id=791dd09f806e4b5ea8185c86741b9731]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.885687 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a48b2fd96ed448fa974e02b9110e2c49. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.885720 618 tablet_bootstrap.cc:654] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.887086 620 tablet_bootstrap.cc:492] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:07.887259 320 consensus_queue.cc:1035] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:07.887782 491 tablet_bootstrap.cc:492] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:07.888088 618 tablet_bootstrap.cc:492] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:07.888157 618 ts_tablet_manager.cc:1367] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:59:07.888562 491 tablet_bootstrap.cc:654] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.888530 618 raft_consensus.cc:348] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:07.888617 618 raft_consensus.cc:374] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.888643 618 raft_consensus.cc:729] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.888778 618 consensus_queue.cc:260] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:07.889150 495 tablet_bootstrap.cc:492] T a48b2fd96ed448fa974e02b9110e2c49 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:07.889535 491 tablet_bootstrap.cc:492] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:07.889611 491 ts_tablet_manager.cc:1367] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:59:07.889799 618 ts_tablet_manager.cc:1398] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:59:07.889927 495 tablet_bootstrap.cc:654] T a48b2fd96ed448fa974e02b9110e2c49 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.889940 618 tablet_bootstrap.cc:492] T b2d4cffdd0f14378804035004ab246b9 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:07.890017 491 raft_consensus.cc:348] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:07.890087 491 raft_consensus.cc:374] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.890106 491 raft_consensus.cc:729] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.890193 491 consensus_queue.cc:260] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:07.890383 491 ts_tablet_manager.cc:1398] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:07.891641 618 tablet_bootstrap.cc:654] T b2d4cffdd0f14378804035004ab246b9 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.892275 22959 tablet_service.cc:1467] Processing CreateTablet for tablet b2d4cffdd0f14378804035004ab246b9 (DEFAULT_TABLE table=rebalancer_extra_table_68 [id=791dd09f806e4b5ea8185c86741b9731]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.892517 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b2d4cffdd0f14378804035004ab246b9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.893242 495 tablet_bootstrap.cc:492] T a48b2fd96ed448fa974e02b9110e2c49 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:07.893323 495 ts_tablet_manager.cc:1367] T a48b2fd96ed448fa974e02b9110e2c49 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:59:07.893669 495 raft_consensus.cc:348] T a48b2fd96ed448fa974e02b9110e2c49 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:07.893749 495 raft_consensus.cc:374] T a48b2fd96ed448fa974e02b9110e2c49 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.893775 495 raft_consensus.cc:729] T a48b2fd96ed448fa974e02b9110e2c49 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.893864 495 consensus_queue.cc:260] T a48b2fd96ed448fa974e02b9110e2c49 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:07.894104 495 ts_tablet_manager.cc:1398] T a48b2fd96ed448fa974e02b9110e2c49 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:07.895228 620 tablet_bootstrap.cc:654] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.895429 625 tablet_bootstrap.cc:492] T b2d4cffdd0f14378804035004ab246b9 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:07.896553 618 tablet_bootstrap.cc:492] T b2d4cffdd0f14378804035004ab246b9 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:07.896629 618 ts_tablet_manager.cc:1367] T b2d4cffdd0f14378804035004ab246b9 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.001s
I20240502 07:59:07.897042 618 raft_consensus.cc:348] T b2d4cffdd0f14378804035004ab246b9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:07.897125 618 raft_consensus.cc:374] T b2d4cffdd0f14378804035004ab246b9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.897156 618 raft_consensus.cc:729] T b2d4cffdd0f14378804035004ab246b9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.897239 618 consensus_queue.cc:260] T b2d4cffdd0f14378804035004ab246b9 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:07.897653 22697 tablet_service.cc:1467] Processing CreateTablet for tablet b2d4cffdd0f14378804035004ab246b9 (DEFAULT_TABLE table=rebalancer_extra_table_68 [id=791dd09f806e4b5ea8185c86741b9731]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.897897 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b2d4cffdd0f14378804035004ab246b9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.898483 22695 tablet_service.cc:1467] Processing CreateTablet for tablet a48b2fd96ed448fa974e02b9110e2c49 (DEFAULT_TABLE table=rebalancer_extra_table_68 [id=791dd09f806e4b5ea8185c86741b9731]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.898732 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a48b2fd96ed448fa974e02b9110e2c49. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.898766 625 tablet_bootstrap.cc:654] T b2d4cffdd0f14378804035004ab246b9 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.899921 628 tablet_bootstrap.cc:492] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:07.900000 22696 tablet_service.cc:1467] Processing CreateTablet for tablet 9a3ed793ccf54fc3ba42f9f13db2aa9f (DEFAULT_TABLE table=rebalancer_extra_table_68 [id=791dd09f806e4b5ea8185c86741b9731]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:07.900182 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9a3ed793ccf54fc3ba42f9f13db2aa9f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:07.901005 628 tablet_bootstrap.cc:654] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.903208 618 ts_tablet_manager.cc:1398] T b2d4cffdd0f14378804035004ab246b9 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.007s user 0.000s sys 0.001s
I20240502 07:59:07.903621 320 consensus_queue.cc:1035] T f09319be8bb445839415e70203a53ac1 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:07.904925 628 tablet_bootstrap.cc:492] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:07.905052 628 ts_tablet_manager.cc:1367] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:59:07.905648 628 raft_consensus.cc:348] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:07.905800 628 raft_consensus.cc:374] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.905874 628 raft_consensus.cc:729] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.906034 628 consensus_queue.cc:260] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:07.906322 628 ts_tablet_manager.cc:1398] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:07.907245 628 tablet_bootstrap.cc:492] T a48b2fd96ed448fa974e02b9110e2c49 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:07.908120 628 tablet_bootstrap.cc:654] T a48b2fd96ed448fa974e02b9110e2c49 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.909093 628 tablet_bootstrap.cc:492] T a48b2fd96ed448fa974e02b9110e2c49 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:07.909204 628 ts_tablet_manager.cc:1367] T a48b2fd96ed448fa974e02b9110e2c49 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.001s
I20240502 07:59:07.911078 628 raft_consensus.cc:348] T a48b2fd96ed448fa974e02b9110e2c49 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:07.911214 628 raft_consensus.cc:374] T a48b2fd96ed448fa974e02b9110e2c49 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.911278 628 raft_consensus.cc:729] T a48b2fd96ed448fa974e02b9110e2c49 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.911417 628 consensus_queue.cc:260] T a48b2fd96ed448fa974e02b9110e2c49 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:07.911759 628 ts_tablet_manager.cc:1398] T a48b2fd96ed448fa974e02b9110e2c49 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 07:59:07.911931 628 tablet_bootstrap.cc:492] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:07.912361 620 tablet_bootstrap.cc:492] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:07.912446 620 ts_tablet_manager.cc:1367] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.025s user 0.002s sys 0.000s
I20240502 07:59:07.912784 628 tablet_bootstrap.cc:654] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:07.912923 620 raft_consensus.cc:348] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:07.913025 620 raft_consensus.cc:374] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.913074 620 raft_consensus.cc:729] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.913182 620 consensus_queue.cc:260] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:07.913451 620 ts_tablet_manager.cc:1398] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:07.913934 628 tablet_bootstrap.cc:492] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:07.914013 628 ts_tablet_manager.cc:1367] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:07.914331 628 raft_consensus.cc:348] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:07.914422 628 raft_consensus.cc:374] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.914448 628 raft_consensus.cc:729] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.914537 628 consensus_queue.cc:260] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:07.914770 628 ts_tablet_manager.cc:1398] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:07.915829 32740 raft_consensus.cc:482] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:07.915900 32740 raft_consensus.cc:504] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:07.916198 32740 leader_election.cc:288] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:07.916555 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b2d4cffdd0f14378804035004ab246b9" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
W20240502 07:59:07.917002 22651 leader_election.cc:341] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): Illegal state: must be running to vote when last-logged opid is not known
I20240502 07:59:07.917532 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b2d4cffdd0f14378804035004ab246b9" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:07.917623 25134 raft_consensus.cc:2454] T b2d4cffdd0f14378804035004ab246b9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:59:07.917862 22653 leader_election.cc:302] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 725969d490a04fdfb3ba890c3f5096e3; no voters: 02a92e1c084b4308b9e855a4cc2c295f
I20240502 07:59:07.918030 32740 raft_consensus.cc:2781] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:07.918085 32740 raft_consensus.cc:482] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:07.918118 32740 raft_consensus.cc:3037] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.919003 32740 raft_consensus.cc:504] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:07.919308 32740 leader_election.cc:288] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:07.920957 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b2d4cffdd0f14378804035004ab246b9" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:07.921408 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b2d4cffdd0f14378804035004ab246b9" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:07.921500 25134 raft_consensus.cc:3037] T b2d4cffdd0f14378804035004ab246b9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
W20240502 07:59:07.921532 22651 leader_election.cc:341] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): Illegal state: must be running to vote when last-logged opid is not known
I20240502 07:59:07.921972 625 tablet_bootstrap.cc:492] T b2d4cffdd0f14378804035004ab246b9 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:07.922036 625 ts_tablet_manager.cc:1367] T b2d4cffdd0f14378804035004ab246b9 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.027s user 0.001s sys 0.000s
I20240502 07:59:07.922266 25134 raft_consensus.cc:2454] T b2d4cffdd0f14378804035004ab246b9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:59:07.922389 625 raft_consensus.cc:348] T b2d4cffdd0f14378804035004ab246b9 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:07.922462 625 raft_consensus.cc:374] T b2d4cffdd0f14378804035004ab246b9 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:07.922492 625 raft_consensus.cc:729] T b2d4cffdd0f14378804035004ab246b9 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:07.922534 22653 leader_election.cc:302] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 725969d490a04fdfb3ba890c3f5096e3; no voters: 02a92e1c084b4308b9e855a4cc2c295f
I20240502 07:59:07.922590 625 consensus_queue.cc:260] T b2d4cffdd0f14378804035004ab246b9 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:07.922797 625 ts_tablet_manager.cc:1398] T b2d4cffdd0f14378804035004ab246b9 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:07.922801 32740 raft_consensus.cc:2781] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:07.922871 32740 raft_consensus.cc:686] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:59:07.922978 32740 consensus_queue.cc:237] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
W20240502 07:59:07.924430 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20240502 07:59:07.927686 22444 catalog_manager.cc:5484] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:07.959373 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:59:07.962827 25266 raft_consensus.cc:1261] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:07.962967 22845 raft_consensus.cc:1261] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:07.963361 32236 consensus_queue.cc:1035] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:07.963624 407 consensus_queue.cc:1035] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:07.966612 22444 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet bd57ea57711f4562b514353828fb934e (table rebalancer_dynamic_table [id=53c5a7fa93ae4c6fb04483f33b7bcaae]). Expected version 1 got 0
I20240502 07:59:07.967868 22436 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet bd57ea57711f4562b514353828fb934e (table rebalancer_dynamic_table [id=53c5a7fa93ae4c6fb04483f33b7bcaae]). Expected version 1 got 0
I20240502 07:59:07.968554 641 tablet.cc:1722] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:07.968744 22444 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet bd57ea57711f4562b514353828fb934e (table rebalancer_dynamic_table [id=53c5a7fa93ae4c6fb04483f33b7bcaae]). Expected version 1 got 0
I20240502 07:59:07.968850 641 tablet.cc:1620] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:59:07.969358 642 tablet.cc:1722] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:07.969748 642 tablet.cc:1620] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:59:07.969980 22444 catalog_manager.cc:5576] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769) does not have the latest schema for tablet bd57ea57711f4562b514353828fb934e (table rebalancer_dynamic_table [id=53c5a7fa93ae4c6fb04483f33b7bcaae]). Expected version 1 got 0
I20240502 07:59:07.970618 640 tablet.cc:1722] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:07.971506 22436 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet bd57ea57711f4562b514353828fb934e (table rebalancer_dynamic_table [id=53c5a7fa93ae4c6fb04483f33b7bcaae]). Expected version 1 got 0
I20240502 07:59:07.972313 640 tablet.cc:1620] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:59:07.973589 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=53c5a7fa93ae4c6fb04483f33b7bcaae] alter complete (version 1)
I20240502 07:59:07.995870 458 raft_consensus.cc:482] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:07.995977 458 raft_consensus.cc:504] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:07.996336 458 leader_election.cc:288] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:59:07.996620 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9a3ed793ccf54fc3ba42f9f13db2aa9f" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:07.996616 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9a3ed793ccf54fc3ba42f9f13db2aa9f" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:07.996758 22584 raft_consensus.cc:2454] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:59:07.996758 24871 raft_consensus.cc:2454] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:59:07.996760 312 raft_consensus.cc:482] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:07.996848 312 raft_consensus.cc:504] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:07.996990 22650 leader_election.cc:302] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:07.997140 458 raft_consensus.cc:2781] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:07.997203 458 raft_consensus.cc:482] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:07.997251 312 leader_election.cc:288] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:07.997246 458 raft_consensus.cc:3037] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.997493 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a48b2fd96ed448fa974e02b9110e2c49" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:07.997566 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a48b2fd96ed448fa974e02b9110e2c49" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:07.997607 25266 raft_consensus.cc:2454] T a48b2fd96ed448fa974e02b9110e2c49 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:07.997656 22716 raft_consensus.cc:2454] T a48b2fd96ed448fa974e02b9110e2c49 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:07.997865 25068 leader_election.cc:302] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:07.998028 312 raft_consensus.cc:2781] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:07.998083 312 raft_consensus.cc:482] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:07.998127 312 raft_consensus.cc:3037] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.998198 458 raft_consensus.cc:504] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:07.998452 458 leader_election.cc:288] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:59:07.998716 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9a3ed793ccf54fc3ba42f9f13db2aa9f" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:07.998811 22584 raft_consensus.cc:3037] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.998795 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9a3ed793ccf54fc3ba42f9f13db2aa9f" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:07.998898 24871 raft_consensus.cc:3037] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.999121 312 raft_consensus.cc:504] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:07.999423 312 leader_election.cc:288] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:07.999483 22584 raft_consensus.cc:2454] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:59:07.999739 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a48b2fd96ed448fa974e02b9110e2c49" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:07.999819 22653 leader_election.cc:302] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:07.999842 22716 raft_consensus.cc:3037] T a48b2fd96ed448fa974e02b9110e2c49 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:07.999886 24871 raft_consensus.cc:2454] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:59:07.999976 458 raft_consensus.cc:2781] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:08.000048 458 raft_consensus.cc:686] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:59:08.000113 25266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a48b2fd96ed448fa974e02b9110e2c49" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:08.000175 458 consensus_queue.cc:237] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:08.000222 25266 raft_consensus.cc:3037] T a48b2fd96ed448fa974e02b9110e2c49 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:08.000758 22716 raft_consensus.cc:2454] T a48b2fd96ed448fa974e02b9110e2c49 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:08.001132 25266 raft_consensus.cc:2454] T a48b2fd96ed448fa974e02b9110e2c49 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:08.001344 25068 leader_election.cc:302] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:08.001797 312 raft_consensus.cc:2781] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:08.002111 312 raft_consensus.cc:686] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:59:08.002250 312 consensus_queue.cc:237] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:08.003158 22436 catalog_manager.cc:5484] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:08.004542 22444 catalog_manager.cc:5484] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
W20240502 07:59:08.027657 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:08.055850 438 raft_consensus.cc:482] T b2d4cffdd0f14378804035004ab246b9 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:08.055959 438 raft_consensus.cc:504] T b2d4cffdd0f14378804035004ab246b9 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:08.056282 438 leader_election.cc:288] T b2d4cffdd0f14378804035004ab246b9 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:08.056565 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b2d4cffdd0f14378804035004ab246b9" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:08.056739 25134 raft_consensus.cc:2379] T b2d4cffdd0f14378804035004ab246b9 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 02a92e1c084b4308b9e855a4cc2c295f in current term 1: Already voted for candidate 725969d490a04fdfb3ba890c3f5096e3 in this term.
I20240502 07:59:08.056691 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b2d4cffdd0f14378804035004ab246b9" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:08.057473 22912 leader_election.cc:302] T b2d4cffdd0f14378804035004ab246b9 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f; no voters: 4654231dfe994c1a9d2e37d9c3d75699, 725969d490a04fdfb3ba890c3f5096e3
I20240502 07:59:08.057716 438 raft_consensus.cc:3037] T b2d4cffdd0f14378804035004ab246b9 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:08.058719 438 raft_consensus.cc:2726] T b2d4cffdd0f14378804035004ab246b9 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20240502 07:59:08.066618 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:08.077311 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:59:08.082289 640 tablet.cc:1722] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:08.082571 640 tablet.cc:1620] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:59:08.082571 642 tablet.cc:1722] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:08.082836 642 tablet.cc:1620] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:59:08.082801 641 tablet.cc:1722] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:08.083039 641 tablet.cc:1620] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69: MemRowSet was empty: no flush needed.
I20240502 07:59:08.088040 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=53c5a7fa93ae4c6fb04483f33b7bcaae] alter complete (version 2)
W20240502 07:59:08.098973 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:08.190935 22438 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:59:08.196255 641 tablet.cc:1722] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:08.197389 640 tablet.cc:1722] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:08.201381 22438 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=53c5a7fa93ae4c6fb04483f33b7bcaae] alter complete (version 3)
I20240502 07:59:08.201658 642 tablet.cc:1722] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:08.303748 22444 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:08.303900 22444 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:08.305936 22444 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet bd57ea57711f4562b514353828fb934e
I20240502 07:59:08.306375 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet bd57ea57711f4562b514353828fb934e with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:08 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:59:08.306550 25246 tablet_service.cc:1514] Processing DeleteTablet for tablet bd57ea57711f4562b514353828fb934e with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:08 UTC) from {username='slave'} at 127.0.0.1:33118
I20240502 07:59:08.306550 22826 tablet_service.cc:1514] Processing DeleteTablet for tablet bd57ea57711f4562b514353828fb934e with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:08 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:59:08.306721 707 tablet_replica.cc:330] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:59:08.306831 709 tablet_replica.cc:330] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69: stopping tablet replica
I20240502 07:59:08.306926 707 raft_consensus.cc:2229] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:59:08.306962 709 raft_consensus.cc:2229] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:08.307076 709 raft_consensus.cc:2258] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:08.307140 707 raft_consensus.cc:2258] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:08.307610 709 ts_tablet_manager.cc:1875] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:08.307627 707 ts_tablet_manager.cc:1875] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:08.309682 709 ts_tablet_manager.cc:1888] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:08.309747 709 log.cc:1196] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-7/wal/wals/bd57ea57711f4562b514353828fb934e
I20240502 07:59:08.309926 707 ts_tablet_manager.cc:1888] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:08.310003 709 ts_tablet_manager.cc:1909] T bd57ea57711f4562b514353828fb934e P 7eced220cf1642f3873fc385b6191e69: Deleting consensus metadata
I20240502 07:59:08.310011 707 log.cc:1196] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/bd57ea57711f4562b514353828fb934e
I20240502 07:59:08.310292 707 ts_tablet_manager.cc:1909] T bd57ea57711f4562b514353828fb934e P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:59:08.310611 643 consensus_queue.cc:1035] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:08.310690 708 tablet_replica.cc:330] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:59:08.310716 22430 catalog_manager.cc:4830] TS 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): tablet bd57ea57711f4562b514353828fb934e (table rebalancer_dynamic_table_new_name [id=53c5a7fa93ae4c6fb04483f33b7bcaae]) successfully deleted
I20240502 07:59:08.310809 708 raft_consensus.cc:2229] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:08.310950 708 raft_consensus.cc:2258] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:08.311002 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet bd57ea57711f4562b514353828fb934e (table rebalancer_dynamic_table_new_name [id=53c5a7fa93ae4c6fb04483f33b7bcaae]) successfully deleted
I20240502 07:59:08.311481 708 ts_tablet_manager.cc:1875] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:08.313741 708 ts_tablet_manager.cc:1888] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:08.313849 708 log.cc:1196] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/bd57ea57711f4562b514353828fb934e
I20240502 07:59:08.314181 708 ts_tablet_manager.cc:1909] T bd57ea57711f4562b514353828fb934e P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:59:08.315104 643 consensus_queue.cc:1035] T b2d4cffdd0f14378804035004ab246b9 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:08.315685 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet bd57ea57711f4562b514353828fb934e (table rebalancer_dynamic_table_new_name [id=53c5a7fa93ae4c6fb04483f33b7bcaae]) successfully deleted
W20240502 07:59:08.371630 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:08.407444 22438 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:59:08.411901 24852 tablet_service.cc:1467] Processing CreateTablet for tablet ac37a02220074211bd34e90ffb72a424 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=e8a7b5ceb3894233a24c585d86793414]), partition=
I20240502 07:59:08.411928 22826 tablet_service.cc:1467] Processing CreateTablet for tablet ac37a02220074211bd34e90ffb72a424 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=e8a7b5ceb3894233a24c585d86793414]), partition=
I20240502 07:59:08.412146 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ac37a02220074211bd34e90ffb72a424. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:08.412151 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ac37a02220074211bd34e90ffb72a424. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:08.412889 22564 tablet_service.cc:1467] Processing CreateTablet for tablet ac37a02220074211bd34e90ffb72a424 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=e8a7b5ceb3894233a24c585d86793414]), partition=
I20240502 07:59:08.413079 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ac37a02220074211bd34e90ffb72a424. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:08.414531 713 tablet_bootstrap.cc:492] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:08.415611 714 tablet_bootstrap.cc:492] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:08.416460 714 tablet_bootstrap.cc:654] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:08.417001 713 tablet_bootstrap.cc:654] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:08.417780 714 tablet_bootstrap.cc:492] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:08.417888 714 ts_tablet_manager.cc:1367] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.002s
I20240502 07:59:08.418123 713 tablet_bootstrap.cc:492] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:08.418200 713 ts_tablet_manager.cc:1367] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.000s
I20240502 07:59:08.418295 714 raft_consensus.cc:348] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:08.418393 714 raft_consensus.cc:374] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:08.418417 714 raft_consensus.cc:729] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:08.418540 714 consensus_queue.cc:260] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:08.418570 713 raft_consensus.cc:348] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:08.418659 713 raft_consensus.cc:374] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:08.418684 713 raft_consensus.cc:729] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:08.418772 713 consensus_queue.cc:260] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:08.419024 713 ts_tablet_manager.cc:1398] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:08.418896 714 ts_tablet_manager.cc:1398] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:08.424857 715 tablet_bootstrap.cc:492] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:08.425734 715 tablet_bootstrap.cc:654] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:08.426654 715 tablet_bootstrap.cc:492] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:08.426759 715 ts_tablet_manager.cc:1367] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
W20240502 07:59:08.427110 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:08.427165 715 raft_consensus.cc:348] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:08.427314 715 raft_consensus.cc:374] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:08.427356 715 raft_consensus.cc:729] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:08.427464 715 consensus_queue.cc:260] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:08.427718 715 ts_tablet_manager.cc:1398] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:08.452325 406 raft_consensus.cc:482] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:08.452418 406 raft_consensus.cc:504] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:08.452858 406 leader_election.cc:288] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:08.453022 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ac37a02220074211bd34e90ffb72a424" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:08.453136 24870 raft_consensus.cc:2454] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:08.453218 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ac37a02220074211bd34e90ffb72a424" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:08.453328 22847 raft_consensus.cc:2454] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:08.453362 22519 leader_election.cc:302] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:08.453501 406 raft_consensus.cc:2781] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:08.453562 406 raft_consensus.cc:482] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:08.453604 406 raft_consensus.cc:3037] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:08.454349 406 raft_consensus.cc:504] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:08.454595 406 leader_election.cc:288] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:08.454855 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ac37a02220074211bd34e90ffb72a424" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:08.454979 24870 raft_consensus.cc:3037] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:08.455140 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ac37a02220074211bd34e90ffb72a424" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:59:08.455255 22847 raft_consensus.cc:3037] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:08.455904 24870 raft_consensus.cc:2454] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:08.456079 22847 raft_consensus.cc:2454] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:08.456177 22519 leader_election.cc:302] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:08.456337 406 raft_consensus.cc:2781] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:08.456411 406 raft_consensus.cc:686] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:59:08.456532 406 consensus_queue.cc:237] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:08.458485 22438 catalog_manager.cc:5484] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } }
I20240502 07:59:08.502629 458 consensus_queue.cc:1035] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:08.502732 320 consensus_queue.cc:1035] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:08.506739 505 consensus_queue.cc:1035] T a48b2fd96ed448fa974e02b9110e2c49 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:08.509959 643 consensus_queue.cc:1035] T 9a3ed793ccf54fc3ba42f9f13db2aa9f P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:08.514461 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_70"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 07:59:08.516382 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_DELETED RPC failed for tablet 41284d865ebb4cd18f2ce08f66343262: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:08.522265 25246 tablet_service.cc:1467] Processing CreateTablet for tablet 0bec41e0dad346beb70667c3894ddeef (DEFAULT_TABLE table=rebalancer_extra_table_70 [id=2e064a3c3c8942a299758a31ce7923d9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:08.522531 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0bec41e0dad346beb70667c3894ddeef. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:08.522579 25235 tablet_service.cc:1467] Processing CreateTablet for tablet b3629284ba374d5c92c6645644c2798d (DEFAULT_TABLE table=rebalancer_extra_table_70 [id=2e064a3c3c8942a299758a31ce7923d9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:08.522748 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b3629284ba374d5c92c6645644c2798d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:08.522826 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 0bec41e0dad346beb70667c3894ddeef (DEFAULT_TABLE table=rebalancer_extra_table_70 [id=2e064a3c3c8942a299758a31ce7923d9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:08.523062 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0bec41e0dad346beb70667c3894ddeef. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:08.523388 22564 tablet_service.cc:1467] Processing CreateTablet for tablet b3629284ba374d5c92c6645644c2798d (DEFAULT_TABLE table=rebalancer_extra_table_70 [id=2e064a3c3c8942a299758a31ce7923d9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:08.523397 22826 tablet_service.cc:1467] Processing CreateTablet for tablet b3629284ba374d5c92c6645644c2798d (DEFAULT_TABLE table=rebalancer_extra_table_70 [id=2e064a3c3c8942a299758a31ce7923d9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:08.523617 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b3629284ba374d5c92c6645644c2798d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:08.523638 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b3629284ba374d5c92c6645644c2798d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:08.524998 22827 tablet_service.cc:1467] Processing CreateTablet for tablet adcce10f0dae4dacaea48fd3721ec5d6 (DEFAULT_TABLE table=rebalancer_extra_table_70 [id=2e064a3c3c8942a299758a31ce7923d9]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:08.525096 714 tablet_bootstrap.cc:492] T 0bec41e0dad346beb70667c3894ddeef P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:08.525203 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet adcce10f0dae4dacaea48fd3721ec5d6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:08.525719 715 tablet_bootstrap.cc:492] T b3629284ba374d5c92c6645644c2798d P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:08.526254 714 tablet_bootstrap.cc:654] T 0bec41e0dad346beb70667c3894ddeef P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:08.526218 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 0bec41e0dad346beb70667c3894ddeef (DEFAULT_TABLE table=rebalancer_extra_table_70 [id=2e064a3c3c8942a299758a31ce7923d9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:08.526449 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0bec41e0dad346beb70667c3894ddeef. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:08.526611 715 tablet_bootstrap.cc:654] T b3629284ba374d5c92c6645644c2798d P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:08.526784 713 tablet_bootstrap.cc:492] T b3629284ba374d5c92c6645644c2798d P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:08.527420 22957 tablet_service.cc:1467] Processing CreateTablet for tablet adcce10f0dae4dacaea48fd3721ec5d6 (DEFAULT_TABLE table=rebalancer_extra_table_70 [id=2e064a3c3c8942a299758a31ce7923d9]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:08.527541 713 tablet_bootstrap.cc:654] T b3629284ba374d5c92c6645644c2798d P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:08.527578 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet adcce10f0dae4dacaea48fd3721ec5d6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:08.528465 713 tablet_bootstrap.cc:492] T b3629284ba374d5c92c6645644c2798d P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:08.528534 713 ts_tablet_manager.cc:1367] T b3629284ba374d5c92c6645644c2798d P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:08.528810 715 tablet_bootstrap.cc:492] T b3629284ba374d5c92c6645644c2798d P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:08.528875 715 ts_tablet_manager.cc:1367] T b3629284ba374d5c92c6645644c2798d P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:59:08.528928 713 raft_consensus.cc:348] T b3629284ba374d5c92c6645644c2798d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.529022 713 raft_consensus.cc:374] T b3629284ba374d5c92c6645644c2798d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:08.529049 713 raft_consensus.cc:729] T b3629284ba374d5c92c6645644c2798d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:08.529147 713 consensus_queue.cc:260] T b3629284ba374d5c92c6645644c2798d P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.529317 715 raft_consensus.cc:348] T b3629284ba374d5c92c6645644c2798d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.529407 715 raft_consensus.cc:374] T b3629284ba374d5c92c6645644c2798d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:08.529405 713 ts_tablet_manager.cc:1398] T b3629284ba374d5c92c6645644c2798d P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:08.529434 715 raft_consensus.cc:729] T b3629284ba374d5c92c6645644c2798d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:08.529556 713 tablet_bootstrap.cc:492] T adcce10f0dae4dacaea48fd3721ec5d6 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:08.529527 715 consensus_queue.cc:260] T b3629284ba374d5c92c6645644c2798d P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.529817 715 ts_tablet_manager.cc:1398] T b3629284ba374d5c92c6645644c2798d P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:08.530326 713 tablet_bootstrap.cc:654] T adcce10f0dae4dacaea48fd3721ec5d6 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:08.531077 752 tablet_bootstrap.cc:492] T 0bec41e0dad346beb70667c3894ddeef P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:08.531502 25235 tablet_service.cc:1467] Processing CreateTablet for tablet adcce10f0dae4dacaea48fd3721ec5d6 (DEFAULT_TABLE table=rebalancer_extra_table_70 [id=2e064a3c3c8942a299758a31ce7923d9]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:08.531710 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet adcce10f0dae4dacaea48fd3721ec5d6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:08.532048 752 tablet_bootstrap.cc:654] T 0bec41e0dad346beb70667c3894ddeef P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:08.533053 752 tablet_bootstrap.cc:492] T 0bec41e0dad346beb70667c3894ddeef P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:08.533126 752 ts_tablet_manager.cc:1367] T 0bec41e0dad346beb70667c3894ddeef P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:08.533545 752 raft_consensus.cc:348] T 0bec41e0dad346beb70667c3894ddeef P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:08.533625 752 raft_consensus.cc:374] T 0bec41e0dad346beb70667c3894ddeef P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:08.533650 752 raft_consensus.cc:729] T 0bec41e0dad346beb70667c3894ddeef P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:08.533754 752 consensus_queue.cc:260] T 0bec41e0dad346beb70667c3894ddeef P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:08.534005 752 ts_tablet_manager.cc:1398] T 0bec41e0dad346beb70667c3894ddeef P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:08.534155 752 tablet_bootstrap.cc:492] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:08.534415 748 tablet_bootstrap.cc:492] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:08.534940 752 tablet_bootstrap.cc:654] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:08.535938 748 tablet_bootstrap.cc:654] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:08.536191 713 tablet_bootstrap.cc:492] T adcce10f0dae4dacaea48fd3721ec5d6 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:08.536298 713 ts_tablet_manager.cc:1367] T adcce10f0dae4dacaea48fd3721ec5d6 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
I20240502 07:59:08.536706 713 raft_consensus.cc:348] T adcce10f0dae4dacaea48fd3721ec5d6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.536962 714 tablet_bootstrap.cc:492] T 0bec41e0dad346beb70667c3894ddeef P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:08.537035 714 ts_tablet_manager.cc:1367] T 0bec41e0dad346beb70667c3894ddeef P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.012s user 0.000s sys 0.002s
I20240502 07:59:08.537439 714 raft_consensus.cc:348] T 0bec41e0dad346beb70667c3894ddeef P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:08.537544 714 raft_consensus.cc:374] T 0bec41e0dad346beb70667c3894ddeef P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:08.537575 714 raft_consensus.cc:729] T 0bec41e0dad346beb70667c3894ddeef P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:08.537660 714 consensus_queue.cc:260] T 0bec41e0dad346beb70667c3894ddeef P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:08.537807 713 raft_consensus.cc:374] T adcce10f0dae4dacaea48fd3721ec5d6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:08.537853 713 raft_consensus.cc:729] T adcce10f0dae4dacaea48fd3721ec5d6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:08.537937 714 ts_tablet_manager.cc:1398] T 0bec41e0dad346beb70667c3894ddeef P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:08.537947 713 consensus_queue.cc:260] T adcce10f0dae4dacaea48fd3721ec5d6 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.538261 713 ts_tablet_manager.cc:1398] T adcce10f0dae4dacaea48fd3721ec5d6 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:08.539482 752 tablet_bootstrap.cc:492] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:08.539544 752 ts_tablet_manager.cc:1367] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.000s
I20240502 07:59:08.539880 752 raft_consensus.cc:348] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.539963 752 raft_consensus.cc:374] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:08.539996 752 raft_consensus.cc:729] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:08.540088 752 consensus_queue.cc:260] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.540386 752 ts_tablet_manager.cc:1398] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:08.541054 748 tablet_bootstrap.cc:492] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:08.541213 748 ts_tablet_manager.cc:1367] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
I20240502 07:59:08.541690 748 raft_consensus.cc:348] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.541796 748 raft_consensus.cc:374] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:08.541857 748 raft_consensus.cc:729] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:08.541986 748 consensus_queue.cc:260] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.542359 748 ts_tablet_manager.cc:1398] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
W20240502 07:59:08.542565 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:08.543033 748 tablet_bootstrap.cc:492] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:08.544066 438 raft_consensus.cc:482] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:08.544142 438 raft_consensus.cc:504] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.544173 748 tablet_bootstrap.cc:654] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:08.544521 438 leader_election.cc:288] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:08.545126 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "adcce10f0dae4dacaea48fd3721ec5d6" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:08.545367 748 tablet_bootstrap.cc:492] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:08.545439 748 ts_tablet_manager.cc:1367] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.002s
W20240502 07:59:08.545475 22914 leader_election.cc:341] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Illegal state: must be running to vote when last-logged opid is not known
I20240502 07:59:08.545897 748 raft_consensus.cc:348] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:08.545979 748 raft_consensus.cc:374] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:08.546016 748 raft_consensus.cc:729] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:08.546118 748 consensus_queue.cc:260] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:08.546159 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "adcce10f0dae4dacaea48fd3721ec5d6" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:08.546247 22847 raft_consensus.cc:2454] T adcce10f0dae4dacaea48fd3721ec5d6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:59:08.546360 748 ts_tablet_manager.cc:1398] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:08.546486 748 tablet_bootstrap.cc:492] T adcce10f0dae4dacaea48fd3721ec5d6 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:08.546479 22915 leader_election.cc:302] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters: 7eced220cf1642f3873fc385b6191e69
I20240502 07:59:08.546654 438 raft_consensus.cc:2781] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:08.546696 438 raft_consensus.cc:482] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:08.546732 438 raft_consensus.cc:3037] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:08.547251 748 tablet_bootstrap.cc:654] T adcce10f0dae4dacaea48fd3721ec5d6 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:08.548079 438 raft_consensus.cc:504] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.548377 438 leader_election.cc:288] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:08.548559 748 tablet_bootstrap.cc:492] T adcce10f0dae4dacaea48fd3721ec5d6 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:08.548602 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "adcce10f0dae4dacaea48fd3721ec5d6" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:59:08.548707 22847 raft_consensus.cc:3037] T adcce10f0dae4dacaea48fd3721ec5d6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:08.548559 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "adcce10f0dae4dacaea48fd3721ec5d6" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
W20240502 07:59:08.549196 22914 leader_election.cc:341] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Illegal state: must be running to vote when last-logged opid is not known
I20240502 07:59:08.549512 22847 raft_consensus.cc:2454] T adcce10f0dae4dacaea48fd3721ec5d6 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:59:08.549600 748 ts_tablet_manager.cc:1367] T adcce10f0dae4dacaea48fd3721ec5d6 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240502 07:59:08.549881 22915 leader_election.cc:302] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters: 7eced220cf1642f3873fc385b6191e69
I20240502 07:59:08.549950 748 raft_consensus.cc:348] T adcce10f0dae4dacaea48fd3721ec5d6 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.550072 748 raft_consensus.cc:374] T adcce10f0dae4dacaea48fd3721ec5d6 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:08.550108 748 raft_consensus.cc:729] T adcce10f0dae4dacaea48fd3721ec5d6 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:08.550164 438 raft_consensus.cc:2781] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:08.550207 748 consensus_queue.cc:260] T adcce10f0dae4dacaea48fd3721ec5d6 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.550242 438 raft_consensus.cc:686] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:59:08.550351 438 consensus_queue.cc:237] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.550431 748 ts_tablet_manager.cc:1398] T adcce10f0dae4dacaea48fd3721ec5d6 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:08.552443 22440 catalog_manager.cc:5484] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:08.567569 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:59:08.571385 24870 raft_consensus.cc:1261] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:08.571401 22847 raft_consensus.cc:1261] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:08.571846 406 consensus_queue.cc:1035] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:08.572084 406 consensus_queue.cc:1035] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:08.574854 22444 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet ac37a02220074211bd34e90ffb72a424 (table rebalancer_dynamic_table [id=e8a7b5ceb3894233a24c585d86793414]). Expected version 1 got 0
W20240502 07:59:08.575505 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:08.575593 22440 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet ac37a02220074211bd34e90ffb72a424 (table rebalancer_dynamic_table [id=e8a7b5ceb3894233a24c585d86793414]). Expected version 1 got 0
I20240502 07:59:08.576318 772 tablet.cc:1722] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:08.576567 772 tablet.cc:1620] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:59:08.577869 22440 catalog_manager.cc:5576] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451) does not have the latest schema for tablet ac37a02220074211bd34e90ffb72a424 (table rebalancer_dynamic_table [id=e8a7b5ceb3894233a24c585d86793414]). Expected version 1 got 0
I20240502 07:59:08.580080 642 tablet.cc:1722] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:08.580344 642 tablet.cc:1620] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:59:08.580443 22440 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet ac37a02220074211bd34e90ffb72a424 (table rebalancer_dynamic_table [id=e8a7b5ceb3894233a24c585d86793414]). Expected version 1 got 0
I20240502 07:59:08.582026 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=e8a7b5ceb3894233a24c585d86793414] alter complete (version 1)
I20240502 07:59:08.586134 640 tablet.cc:1722] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:08.586411 640 tablet.cc:1620] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
W20240502 07:59:08.627315 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:08.663928 602 raft_consensus.cc:482] T adcce10f0dae4dacaea48fd3721ec5d6 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:08.664029 602 raft_consensus.cc:504] T adcce10f0dae4dacaea48fd3721ec5d6 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.664364 602 leader_election.cc:288] T adcce10f0dae4dacaea48fd3721ec5d6 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:08.664683 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "adcce10f0dae4dacaea48fd3721ec5d6" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:08.664772 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "adcce10f0dae4dacaea48fd3721ec5d6" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:08.664963 22847 raft_consensus.cc:2379] T adcce10f0dae4dacaea48fd3721ec5d6 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 7eced220cf1642f3873fc385b6191e69 in current term 1: Already voted for candidate 02a92e1c084b4308b9e855a4cc2c295f in this term.
I20240502 07:59:08.665205 25202 leader_election.cc:302] T adcce10f0dae4dacaea48fd3721ec5d6 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7eced220cf1642f3873fc385b6191e69; no voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88
I20240502 07:59:08.665347 602 raft_consensus.cc:3037] T adcce10f0dae4dacaea48fd3721ec5d6 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:08.666311 602 raft_consensus.cc:2726] T adcce10f0dae4dacaea48fd3721ec5d6 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20240502 07:59:08.672150 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:08.685638 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:59:08.689829 772 tablet.cc:1722] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:08.690268 640 tablet.cc:1722] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:08.690374 772 tablet.cc:1620] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:59:08.690539 640 tablet.cc:1620] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:59:08.692493 642 tablet.cc:1722] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:08.692790 642 tablet.cc:1620] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:59:08.695755 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=e8a7b5ceb3894233a24c585d86793414] alter complete (version 2)
W20240502 07:59:08.761015 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 07:59:08.791365 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:08.797466 22438 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:59:08.802556 772 tablet.cc:1722] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:08.802953 642 tablet.cc:1722] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:08.805042 32520 raft_consensus.cc:482] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:08.805127 32520 raft_consensus.cc:504] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:08.805492 32520 leader_election.cc:288] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:08.805979 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0bec41e0dad346beb70667c3894ddeef" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:08.806108 22979 raft_consensus.cc:2454] T 0bec41e0dad346beb70667c3894ddeef P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:59:08.806344 25200 leader_election.cc:302] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:08.807499 640 tablet.cc:1722] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:08.808369 602 raft_consensus.cc:2781] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:08.808427 602 raft_consensus.cc:482] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:08.808468 602 raft_consensus.cc:3037] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:08.809410 602 raft_consensus.cc:504] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:08.809515 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0bec41e0dad346beb70667c3894ddeef" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:08.809630 24870 raft_consensus.cc:2454] T 0bec41e0dad346beb70667c3894ddeef P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:59:08.809726 602 leader_election.cc:288] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 07:59:08.809798 602 raft_consensus.cc:482] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:08.809845 602 raft_consensus.cc:504] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.810072 602 leader_election.cc:288] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:08.810341 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3629284ba374d5c92c6645644c2798d" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:08.810433 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3629284ba374d5c92c6645644c2798d" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:08.810467 22584 raft_consensus.cc:2454] T b3629284ba374d5c92c6645644c2798d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:59:08.810554 22847 raft_consensus.cc:2454] T b3629284ba374d5c92c6645644c2798d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:59:08.810747 25202 leader_election.cc:302] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7eced220cf1642f3873fc385b6191e69, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:08.810901 602 raft_consensus.cc:2781] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:08.810958 602 raft_consensus.cc:482] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:08.810993 602 raft_consensus.cc:3037] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:08.811249 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0bec41e0dad346beb70667c3894ddeef" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:08.811348 22979 raft_consensus.cc:3037] T 0bec41e0dad346beb70667c3894ddeef P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:08.811497 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0bec41e0dad346beb70667c3894ddeef" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:08.811597 24870 raft_consensus.cc:3037] T 0bec41e0dad346beb70667c3894ddeef P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:08.811779 602 raft_consensus.cc:504] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.812342 22979 raft_consensus.cc:2454] T 0bec41e0dad346beb70667c3894ddeef P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:59:08.812047 602 leader_election.cc:288] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:08.812520 24870 raft_consensus.cc:2454] T 0bec41e0dad346beb70667c3894ddeef P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:59:08.812835 25199 leader_election.cc:302] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:08.812894 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3629284ba374d5c92c6645644c2798d" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:08.812983 22584 raft_consensus.cc:3037] T b3629284ba374d5c92c6645644c2798d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:08.813020 602 raft_consensus.cc:2781] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:08.813095 602 raft_consensus.cc:686] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:59:08.813253 602 consensus_queue.cc:237] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:08.813567 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b3629284ba374d5c92c6645644c2798d" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:59:08.813673 22847 raft_consensus.cc:3037] T b3629284ba374d5c92c6645644c2798d P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:08.813959 22584 raft_consensus.cc:2454] T b3629284ba374d5c92c6645644c2798d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:59:08.814064 22438 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=e8a7b5ceb3894233a24c585d86793414] alter complete (version 3)
I20240502 07:59:08.814546 22847 raft_consensus.cc:2454] T b3629284ba374d5c92c6645644c2798d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:59:08.814872 25202 leader_election.cc:302] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7eced220cf1642f3873fc385b6191e69, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:08.815141 32520 raft_consensus.cc:2781] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:08.815279 32520 raft_consensus.cc:686] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:59:08.815404 32520 consensus_queue.cc:237] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } }
I20240502 07:59:08.816216 22440 catalog_manager.cc:5484] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:08.820065 22438 catalog_manager.cc:5484] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } }
W20240502 07:59:08.845811 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:08.919574 22438 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:08.919714 22438 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:08.921769 22438 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet ac37a02220074211bd34e90ffb72a424
I20240502 07:59:08.922335 22564 tablet_service.cc:1514] Processing DeleteTablet for tablet ac37a02220074211bd34e90ffb72a424 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:08 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:59:08.922377 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet ac37a02220074211bd34e90ffb72a424 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:08 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:59:08.922731 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet ac37a02220074211bd34e90ffb72a424 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:08 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:59:08.923004 809 tablet_replica.cc:330] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:59:08.923146 809 raft_consensus.cc:2229] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:08.923246 809 raft_consensus.cc:2258] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:08.923712 809 ts_tablet_manager.cc:1875] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:08.925815 808 tablet_replica.cc:330] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:59:08.925906 809 ts_tablet_manager.cc:1888] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:08.926004 809 log.cc:1196] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/ac37a02220074211bd34e90ffb72a424
I20240502 07:59:08.925999 808 raft_consensus.cc:2229] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:59:08.926206 808 raft_consensus.cc:2258] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:08.927461 808 ts_tablet_manager.cc:1875] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:08.927691 809 ts_tablet_manager.cc:1909] T ac37a02220074211bd34e90ffb72a424 P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:59:08.928253 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet ac37a02220074211bd34e90ffb72a424 (table rebalancer_dynamic_table_new_name [id=e8a7b5ceb3894233a24c585d86793414]) successfully deleted
I20240502 07:59:08.929193 810 tablet_replica.cc:330] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:59:08.929350 810 raft_consensus.cc:2229] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:08.929474 810 raft_consensus.cc:2258] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:08.929548 808 ts_tablet_manager.cc:1888] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:08.929605 808 log.cc:1196] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/ac37a02220074211bd34e90ffb72a424
I20240502 07:59:08.929980 808 ts_tablet_manager.cc:1909] T ac37a02220074211bd34e90ffb72a424 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:59:08.930609 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet ac37a02220074211bd34e90ffb72a424 (table rebalancer_dynamic_table_new_name [id=e8a7b5ceb3894233a24c585d86793414]) successfully deleted
I20240502 07:59:08.930660 810 ts_tablet_manager.cc:1875] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:08.932631 810 ts_tablet_manager.cc:1888] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:08.932713 810 log.cc:1196] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/ac37a02220074211bd34e90ffb72a424
I20240502 07:59:08.933027 810 ts_tablet_manager.cc:1909] T ac37a02220074211bd34e90ffb72a424 P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:59:08.933550 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet ac37a02220074211bd34e90ffb72a424 (table rebalancer_dynamic_table_new_name [id=e8a7b5ceb3894233a24c585d86793414]) successfully deleted
W20240502 07:59:08.953891 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20240502 07:59:09.002887 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:09.023661 22438 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:59:09.028308 22957 tablet_service.cc:1467] Processing CreateTablet for tablet f685643a4174404c9476127f7501370b (DEFAULT_TABLE table=rebalancer_dynamic_table [id=1cd0691c88c74ef99e399d6eff035df4]), partition=
I20240502 07:59:09.028347 22696 tablet_service.cc:1467] Processing CreateTablet for tablet f685643a4174404c9476127f7501370b (DEFAULT_TABLE table=rebalancer_dynamic_table [id=1cd0691c88c74ef99e399d6eff035df4]), partition=
I20240502 07:59:09.028440 25115 tablet_service.cc:1467] Processing CreateTablet for tablet f685643a4174404c9476127f7501370b (DEFAULT_TABLE table=rebalancer_dynamic_table [id=1cd0691c88c74ef99e399d6eff035df4]), partition=
I20240502 07:59:09.028590 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f685643a4174404c9476127f7501370b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:09.028594 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f685643a4174404c9476127f7501370b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:09.028620 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f685643a4174404c9476127f7501370b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:09.030581 752 tablet_bootstrap.cc:492] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:09.030642 841 tablet_bootstrap.cc:492] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:09.031597 752 tablet_bootstrap.cc:654] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:09.031610 842 tablet_bootstrap.cc:492] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:09.032109 841 tablet_bootstrap.cc:654] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:09.032495 842 tablet_bootstrap.cc:654] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:09.033677 752 tablet_bootstrap.cc:492] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:09.033756 752 ts_tablet_manager.cc:1367] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:59:09.033987 842 tablet_bootstrap.cc:492] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:09.034047 842 ts_tablet_manager.cc:1367] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:59:09.034147 752 raft_consensus.cc:348] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:09.034245 752 raft_consensus.cc:374] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:09.034277 752 raft_consensus.cc:729] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:09.034371 752 consensus_queue.cc:260] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:09.034415 842 raft_consensus.cc:348] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:09.034507 842 raft_consensus.cc:374] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:09.034534 842 raft_consensus.cc:729] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:09.034540 841 tablet_bootstrap.cc:492] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:09.034657 752 ts_tablet_manager.cc:1398] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:09.034627 842 consensus_queue.cc:260] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:09.034667 841 ts_tablet_manager.cc:1367] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 07:59:09.034906 842 ts_tablet_manager.cc:1398] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:09.035081 841 raft_consensus.cc:348] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:09.035171 841 raft_consensus.cc:374] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:09.035199 841 raft_consensus.cc:729] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:09.035313 841 consensus_queue.cc:260] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:09.035576 841 ts_tablet_manager.cc:1398] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
W20240502 07:59:09.038749 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:09.139261 634 consensus_queue.cc:1035] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:09.142372 438 consensus_queue.cc:1035] T adcce10f0dae4dacaea48fd3721ec5d6 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:09.170243 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:09.208835 602 consensus_queue.cc:1035] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:09.212086 32520 consensus_queue.cc:1035] T 0bec41e0dad346beb70667c3894ddeef P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:09.253060 804 consensus_queue.cc:1035] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:09.256237 804 consensus_queue.cc:1035] T b3629284ba374d5c92c6645644c2798d P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:09.268757 438 raft_consensus.cc:482] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:09.268834 438 raft_consensus.cc:504] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:09.269104 438 leader_election.cc:288] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:09.269318 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f685643a4174404c9476127f7501370b" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:09.269390 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f685643a4174404c9476127f7501370b" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:09.269443 22716 raft_consensus.cc:2454] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:59:09.269495 25134 raft_consensus.cc:2454] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:59:09.269721 22915 leader_election.cc:302] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 4654231dfe994c1a9d2e37d9c3d75699; no voters:
I20240502 07:59:09.269908 438 raft_consensus.cc:2781] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:09.269951 438 raft_consensus.cc:482] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:09.269986 438 raft_consensus.cc:3037] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:09.270797 438 raft_consensus.cc:504] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:09.271072 438 leader_election.cc:288] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:09.271309 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f685643a4174404c9476127f7501370b" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:09.271410 25134 raft_consensus.cc:3037] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:09.271431 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f685643a4174404c9476127f7501370b" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:09.271530 22716 raft_consensus.cc:3037] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:09.272353 25134 raft_consensus.cc:2454] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:59:09.272353 22716 raft_consensus.cc:2454] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:59:09.272591 22915 leader_election.cc:302] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 4654231dfe994c1a9d2e37d9c3d75699; no voters:
I20240502 07:59:09.272732 438 raft_consensus.cc:2781] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:09.272795 438 raft_consensus.cc:686] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:59:09.272910 438 consensus_queue.cc:237] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:09.274957 22444 catalog_manager.cc:5484] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:09.331828 22444 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_72"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:59:09.338212 22564 tablet_service.cc:1467] Processing CreateTablet for tablet 9caed3c626ce410da00ecdf435174d4a (DEFAULT_TABLE table=rebalancer_extra_table_72 [id=12db4a3bcebe450482becbb38442c92f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:09.338313 25235 tablet_service.cc:1467] Processing CreateTablet for tablet e45d327e069a42088a60bc9c6f0ba797 (DEFAULT_TABLE table=rebalancer_extra_table_72 [id=12db4a3bcebe450482becbb38442c92f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:09.338344 22957 tablet_service.cc:1467] Processing CreateTablet for tablet e45d327e069a42088a60bc9c6f0ba797 (DEFAULT_TABLE table=rebalancer_extra_table_72 [id=12db4a3bcebe450482becbb38442c92f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:09.338515 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e45d327e069a42088a60bc9c6f0ba797. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:09.338506 22564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9caed3c626ce410da00ecdf435174d4a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:09.338615 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e45d327e069a42088a60bc9c6f0ba797. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:09.339043 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 99597b95eb3246beb05ff4a07bd7bb84 (DEFAULT_TABLE table=rebalancer_extra_table_72 [id=12db4a3bcebe450482becbb38442c92f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:09.339267 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 99597b95eb3246beb05ff4a07bd7bb84. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:09.339975 22696 tablet_service.cc:1467] Processing CreateTablet for tablet 9caed3c626ce410da00ecdf435174d4a (DEFAULT_TABLE table=rebalancer_extra_table_72 [id=12db4a3bcebe450482becbb38442c92f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:09.340091 22565 tablet_service.cc:1467] Processing CreateTablet for tablet 99597b95eb3246beb05ff4a07bd7bb84 (DEFAULT_TABLE table=rebalancer_extra_table_72 [id=12db4a3bcebe450482becbb38442c92f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:09.340214 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9caed3c626ce410da00ecdf435174d4a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:09.340292 22565 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 99597b95eb3246beb05ff4a07bd7bb84. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:09.342063 752 tablet_bootstrap.cc:492] T e45d327e069a42088a60bc9c6f0ba797 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:09.342133 841 tablet_bootstrap.cc:492] T 9caed3c626ce410da00ecdf435174d4a P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:09.342226 842 tablet_bootstrap.cc:492] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:09.342271 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 9caed3c626ce410da00ecdf435174d4a (DEFAULT_TABLE table=rebalancer_extra_table_72 [id=12db4a3bcebe450482becbb38442c92f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:09.342499 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9caed3c626ce410da00ecdf435174d4a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:09.342952 841 tablet_bootstrap.cc:654] T 9caed3c626ce410da00ecdf435174d4a P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:09.343024 752 tablet_bootstrap.cc:654] T e45d327e069a42088a60bc9c6f0ba797 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:09.343187 842 tablet_bootstrap.cc:654] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:09.343890 841 tablet_bootstrap.cc:492] T 9caed3c626ce410da00ecdf435174d4a P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:09.343955 841 ts_tablet_manager.cc:1367] T 9caed3c626ce410da00ecdf435174d4a P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:09.343966 887 tablet_bootstrap.cc:492] T e45d327e069a42088a60bc9c6f0ba797 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:09.344362 841 raft_consensus.cc:348] T 9caed3c626ce410da00ecdf435174d4a P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.344453 841 raft_consensus.cc:374] T 9caed3c626ce410da00ecdf435174d4a P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:09.344484 841 raft_consensus.cc:729] T 9caed3c626ce410da00ecdf435174d4a P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:09.344543 752 tablet_bootstrap.cc:492] T e45d327e069a42088a60bc9c6f0ba797 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:09.344617 752 ts_tablet_manager.cc:1367] T e45d327e069a42088a60bc9c6f0ba797 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:59:09.344585 841 consensus_queue.cc:260] T 9caed3c626ce410da00ecdf435174d4a P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.344712 22566 tablet_service.cc:1467] Processing CreateTablet for tablet e45d327e069a42088a60bc9c6f0ba797 (DEFAULT_TABLE table=rebalancer_extra_table_72 [id=12db4a3bcebe450482becbb38442c92f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:09.344910 841 ts_tablet_manager.cc:1398] T 9caed3c626ce410da00ecdf435174d4a P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:09.344923 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e45d327e069a42088a60bc9c6f0ba797. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:09.345103 887 tablet_bootstrap.cc:654] T e45d327e069a42088a60bc9c6f0ba797 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:09.345057 752 raft_consensus.cc:348] T e45d327e069a42088a60bc9c6f0ba797 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.345180 752 raft_consensus.cc:374] T e45d327e069a42088a60bc9c6f0ba797 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:09.345216 752 raft_consensus.cc:729] T e45d327e069a42088a60bc9c6f0ba797 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:09.345338 752 consensus_queue.cc:260] T e45d327e069a42088a60bc9c6f0ba797 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.345631 752 ts_tablet_manager.cc:1398] T e45d327e069a42088a60bc9c6f0ba797 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:09.346168 842 tablet_bootstrap.cc:492] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:09.346230 842 ts_tablet_manager.cc:1367] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.001s
I20240502 07:59:09.346621 842 raft_consensus.cc:348] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.346715 842 raft_consensus.cc:374] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:09.346741 842 raft_consensus.cc:729] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:09.346858 842 consensus_queue.cc:260] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.346947 891 tablet_bootstrap.cc:492] T 9caed3c626ce410da00ecdf435174d4a P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:09.347110 842 ts_tablet_manager.cc:1398] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:09.347667 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 99597b95eb3246beb05ff4a07bd7bb84 (DEFAULT_TABLE table=rebalancer_extra_table_72 [id=12db4a3bcebe450482becbb38442c92f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:09.347854 891 tablet_bootstrap.cc:654] T 9caed3c626ce410da00ecdf435174d4a P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:09.347916 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 99597b95eb3246beb05ff4a07bd7bb84. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:09.348839 887 tablet_bootstrap.cc:492] T e45d327e069a42088a60bc9c6f0ba797 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:09.348907 887 ts_tablet_manager.cc:1367] T e45d327e069a42088a60bc9c6f0ba797 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:59:09.349299 887 raft_consensus.cc:348] T e45d327e069a42088a60bc9c6f0ba797 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.349393 887 raft_consensus.cc:374] T e45d327e069a42088a60bc9c6f0ba797 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:09.349418 887 raft_consensus.cc:729] T e45d327e069a42088a60bc9c6f0ba797 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:09.349498 887 consensus_queue.cc:260] T e45d327e069a42088a60bc9c6f0ba797 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.349763 887 ts_tablet_manager.cc:1398] T e45d327e069a42088a60bc9c6f0ba797 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:09.350601 886 tablet_bootstrap.cc:492] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:09.351337 886 tablet_bootstrap.cc:654] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:09.352108 891 tablet_bootstrap.cc:492] T 9caed3c626ce410da00ecdf435174d4a P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:09.352210 891 ts_tablet_manager.cc:1367] T 9caed3c626ce410da00ecdf435174d4a P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.000s
I20240502 07:59:09.352592 891 raft_consensus.cc:348] T 9caed3c626ce410da00ecdf435174d4a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.352721 891 raft_consensus.cc:374] T 9caed3c626ce410da00ecdf435174d4a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:09.352767 891 raft_consensus.cc:729] T 9caed3c626ce410da00ecdf435174d4a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:09.352871 891 consensus_queue.cc:260] T 9caed3c626ce410da00ecdf435174d4a P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.353241 891 ts_tablet_manager.cc:1398] T 9caed3c626ce410da00ecdf435174d4a P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:09.353289 894 tablet_bootstrap.cc:492] T 99597b95eb3246beb05ff4a07bd7bb84 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:09.354126 894 tablet_bootstrap.cc:654] T 99597b95eb3246beb05ff4a07bd7bb84 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:09.356317 886 tablet_bootstrap.cc:492] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:09.356395 886 ts_tablet_manager.cc:1367] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:59:09.356789 886 raft_consensus.cc:348] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.356881 886 raft_consensus.cc:374] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:09.356936 886 raft_consensus.cc:729] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:09.357223 886 consensus_queue.cc:260] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.358435 886 ts_tablet_manager.cc:1398] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:09.358613 886 tablet_bootstrap.cc:492] T 99597b95eb3246beb05ff4a07bd7bb84 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:09.359429 886 tablet_bootstrap.cc:654] T 99597b95eb3246beb05ff4a07bd7bb84 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:09.359927 894 tablet_bootstrap.cc:492] T 99597b95eb3246beb05ff4a07bd7bb84 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:09.360010 894 ts_tablet_manager.cc:1367] T 99597b95eb3246beb05ff4a07bd7bb84 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.000s
I20240502 07:59:09.360483 886 tablet_bootstrap.cc:492] T 99597b95eb3246beb05ff4a07bd7bb84 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:09.360939 894 raft_consensus.cc:348] T 99597b95eb3246beb05ff4a07bd7bb84 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.361050 894 raft_consensus.cc:374] T 99597b95eb3246beb05ff4a07bd7bb84 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:09.361101 894 raft_consensus.cc:729] T 99597b95eb3246beb05ff4a07bd7bb84 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:09.361217 894 consensus_queue.cc:260] T 99597b95eb3246beb05ff4a07bd7bb84 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.361436 886 ts_tablet_manager.cc:1367] T 99597b95eb3246beb05ff4a07bd7bb84 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:59:09.361869 886 raft_consensus.cc:348] T 99597b95eb3246beb05ff4a07bd7bb84 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.362016 886 raft_consensus.cc:374] T 99597b95eb3246beb05ff4a07bd7bb84 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:09.362030 894 ts_tablet_manager.cc:1398] T 99597b95eb3246beb05ff4a07bd7bb84 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:09.362080 886 raft_consensus.cc:729] T 99597b95eb3246beb05ff4a07bd7bb84 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:09.362221 886 consensus_queue.cc:260] T 99597b95eb3246beb05ff4a07bd7bb84 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.362517 886 ts_tablet_manager.cc:1398] T 99597b95eb3246beb05ff4a07bd7bb84 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:09.362655 886 tablet_bootstrap.cc:492] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:09.363379 886 tablet_bootstrap.cc:654] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:09.364387 886 tablet_bootstrap.cc:492] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:09.364490 886 ts_tablet_manager.cc:1367] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:09.364938 886 raft_consensus.cc:348] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.365176 886 raft_consensus.cc:374] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:09.365238 886 raft_consensus.cc:729] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:09.365360 886 consensus_queue.cc:260] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.365628 886 ts_tablet_manager.cc:1398] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:09.417801 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:59:09.421614 22716 raft_consensus.cc:1261] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:09.422058 438 consensus_queue.cc:1035] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:09.422912 25134 raft_consensus.cc:1261] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
W20240502 07:59:09.423173 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:09.423482 438 consensus_queue.cc:1035] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:09.425772 22440 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet f685643a4174404c9476127f7501370b (table rebalancer_dynamic_table [id=1cd0691c88c74ef99e399d6eff035df4]). Expected version 1 got 0
I20240502 07:59:09.427281 22438 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet f685643a4174404c9476127f7501370b (table rebalancer_dynamic_table [id=1cd0691c88c74ef99e399d6eff035df4]). Expected version 1 got 0
I20240502 07:59:09.428715 505 raft_consensus.cc:482] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:09.428794 505 raft_consensus.cc:504] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.429113 505 leader_election.cc:288] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:59:09.429306 22438 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet f685643a4174404c9476127f7501370b (table rebalancer_dynamic_table [id=1cd0691c88c74ef99e399d6eff035df4]). Expected version 1 got 0
I20240502 07:59:09.429575 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99597b95eb3246beb05ff4a07bd7bb84" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:09.429706 22586 raft_consensus.cc:2454] T 99597b95eb3246beb05ff4a07bd7bb84 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:09.429811 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99597b95eb3246beb05ff4a07bd7bb84" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:09.429919 24871 raft_consensus.cc:2454] T 99597b95eb3246beb05ff4a07bd7bb84 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:09.430425 22438 catalog_manager.cc:5576] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205) does not have the latest schema for tablet f685643a4174404c9476127f7501370b (table rebalancer_dynamic_table [id=1cd0691c88c74ef99e399d6eff035df4]). Expected version 1 got 0
I20240502 07:59:09.430428 25068 leader_election.cc:302] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:09.430995 505 raft_consensus.cc:2781] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:09.431049 505 raft_consensus.cc:482] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:09.431085 505 raft_consensus.cc:3037] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:09.431105 905 tablet.cc:1722] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:09.431381 905 tablet.cc:1620] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:59:09.432081 505 raft_consensus.cc:504] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.432343 505 leader_election.cc:288] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:59:09.432721 22586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99597b95eb3246beb05ff4a07bd7bb84" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:09.432811 22586 raft_consensus.cc:3037] T 99597b95eb3246beb05ff4a07bd7bb84 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:09.433665 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99597b95eb3246beb05ff4a07bd7bb84" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:09.433765 22586 raft_consensus.cc:2454] T 99597b95eb3246beb05ff4a07bd7bb84 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:09.433776 24871 raft_consensus.cc:3037] T 99597b95eb3246beb05ff4a07bd7bb84 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:09.434034 25071 leader_election.cc:302] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:09.434702 24871 raft_consensus.cc:2454] T 99597b95eb3246beb05ff4a07bd7bb84 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
I20240502 07:59:09.435938 505 raft_consensus.cc:2781] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:09.436033 505 raft_consensus.cc:686] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:59:09.436142 505 consensus_queue.cc:237] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.438158 904 tablet.cc:1722] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:09.438793 904 tablet.cc:1620] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:09.439337 22438 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet f685643a4174404c9476127f7501370b (table rebalancer_dynamic_table [id=1cd0691c88c74ef99e399d6eff035df4]). Expected version 1 got 0
I20240502 07:59:09.439725 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=1cd0691c88c74ef99e399d6eff035df4] alter complete (version 1)
I20240502 07:59:09.440147 907 tablet.cc:1722] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:09.440390 907 tablet.cc:1620] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:59:09.440495 22444 catalog_manager.cc:5576] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387) does not have the latest schema for tablet f685643a4174404c9476127f7501370b (table rebalancer_dynamic_table [id=1cd0691c88c74ef99e399d6eff035df4]). Expected version 1 got 0
I20240502 07:59:09.443691 22444 catalog_manager.cc:5484] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:09.507789 719 raft_consensus.cc:482] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:09.507926 719 raft_consensus.cc:504] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.508293 719 leader_election.cc:288] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:09.508582 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9caed3c626ce410da00ecdf435174d4a" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:09.508719 22847 raft_consensus.cc:2454] T 9caed3c626ce410da00ecdf435174d4a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:09.508945 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9caed3c626ce410da00ecdf435174d4a" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:09.508991 22522 leader_election.cc:302] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:09.509075 22716 raft_consensus.cc:2454] T 9caed3c626ce410da00ecdf435174d4a P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:09.509219 719 raft_consensus.cc:2781] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:09.509280 719 raft_consensus.cc:482] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:09.509312 719 raft_consensus.cc:3037] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:09.510103 719 raft_consensus.cc:504] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.510402 719 leader_election.cc:288] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:09.510762 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9caed3c626ce410da00ecdf435174d4a" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:59:09.510890 22845 raft_consensus.cc:3037] T 9caed3c626ce410da00ecdf435174d4a P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:09.510977 22716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9caed3c626ce410da00ecdf435174d4a" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:09.511081 22716 raft_consensus.cc:3037] T 9caed3c626ce410da00ecdf435174d4a P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:09.511929 22845 raft_consensus.cc:2454] T 9caed3c626ce410da00ecdf435174d4a P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:09.512128 22716 raft_consensus.cc:2454] T 9caed3c626ce410da00ecdf435174d4a P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:09.512306 22522 leader_election.cc:302] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:09.512565 719 raft_consensus.cc:2781] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:09.512645 719 raft_consensus.cc:686] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:59:09.512790 719 consensus_queue.cc:237] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.515070 22438 catalog_manager.cc:5484] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } }
W20240502 07:59:09.535256 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 07:59:09.540591 22438 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:59:09.545887 904 tablet.cc:1722] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:09.546258 904 tablet.cc:1620] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:09.546628 907 tablet.cc:1722] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:09.546625 905 tablet.cc:1722] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:09.546918 907 tablet.cc:1620] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3: MemRowSet was empty: no flush needed.
I20240502 07:59:09.546926 905 tablet.cc:1620] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699: MemRowSet was empty: no flush needed.
I20240502 07:59:09.552023 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=1cd0691c88c74ef99e399d6eff035df4] alter complete (version 2)
I20240502 07:59:09.653851 22438 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:59:09.660378 904 tablet.cc:1722] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:09.661551 905 tablet.cc:1722] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:09.662446 907 tablet.cc:1722] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:09.676689 22438 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=1cd0691c88c74ef99e399d6eff035df4] alter complete (version 3)
W20240502 07:59:09.715150 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:09.747860 719 raft_consensus.cc:482] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:09.747972 719 raft_consensus.cc:504] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.748345 719 leader_election.cc:288] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:09.748689 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e45d327e069a42088a60bc9c6f0ba797" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:09.748765 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e45d327e069a42088a60bc9c6f0ba797" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:09.748831 22978 raft_consensus.cc:2454] T e45d327e069a42088a60bc9c6f0ba797 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:09.748878 25265 raft_consensus.cc:2454] T e45d327e069a42088a60bc9c6f0ba797 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:09.749109 22520 leader_election.cc:302] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:09.749419 719 raft_consensus.cc:2781] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:09.749482 719 raft_consensus.cc:482] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:09.749524 719 raft_consensus.cc:3037] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:09.750614 719 raft_consensus.cc:504] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.750880 719 leader_election.cc:288] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 07:59:09.751181 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e45d327e069a42088a60bc9c6f0ba797" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:09.751217 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e45d327e069a42088a60bc9c6f0ba797" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:09.751277 22978 raft_consensus.cc:3037] T e45d327e069a42088a60bc9c6f0ba797 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:09.751329 25265 raft_consensus.cc:3037] T e45d327e069a42088a60bc9c6f0ba797 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:09.752221 22978 raft_consensus.cc:2454] T e45d327e069a42088a60bc9c6f0ba797 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:09.752285 25265 raft_consensus.cc:2454] T e45d327e069a42088a60bc9c6f0ba797 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:09.752435 22520 leader_election.cc:302] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:09.752681 719 raft_consensus.cc:2781] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:09.752764 719 raft_consensus.cc:686] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:59:09.752882 719 consensus_queue.cc:237] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:09.755144 22440 catalog_manager.cc:5484] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } }
W20240502 07:59:09.773515 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 07:59:09.779193 22440 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:09.779325 22440 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:09.781366 22440 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet f685643a4174404c9476127f7501370b
I20240502 07:59:09.781843 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet f685643a4174404c9476127f7501370b with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:09 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:59:09.782164 22696 tablet_service.cc:1514] Processing DeleteTablet for tablet f685643a4174404c9476127f7501370b with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:09 UTC) from {username='slave'} at 127.0.0.1:42244
I20240502 07:59:09.782212 971 tablet_replica.cc:330] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:59:09.782373 971 raft_consensus.cc:2229] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:59:09.782410 25115 tablet_service.cc:1514] Processing DeleteTablet for tablet f685643a4174404c9476127f7501370b with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:09 UTC) from {username='slave'} at 127.0.0.1:42370
I20240502 07:59:09.782585 971 raft_consensus.cc:2258] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:09.782655 972 tablet_replica.cc:330] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3: stopping tablet replica
I20240502 07:59:09.782868 972 raft_consensus.cc:2229] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:09.782923 973 tablet_replica.cc:330] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699: stopping tablet replica
I20240502 07:59:09.782992 972 raft_consensus.cc:2258] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:09.783068 973 raft_consensus.cc:2229] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:09.783106 971 ts_tablet_manager.cc:1875] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:09.783192 973 raft_consensus.cc:2258] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:09.783438 972 ts_tablet_manager.cc:1875] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:09.783717 973 ts_tablet_manager.cc:1875] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:09.785310 972 ts_tablet_manager.cc:1888] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:09.785374 972 log.cc:1196] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-1/wal/wals/f685643a4174404c9476127f7501370b
I20240502 07:59:09.785594 971 ts_tablet_manager.cc:1888] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:09.785701 971 log.cc:1196] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/f685643a4174404c9476127f7501370b
I20240502 07:59:09.785728 972 ts_tablet_manager.cc:1909] T f685643a4174404c9476127f7501370b P 725969d490a04fdfb3ba890c3f5096e3: Deleting consensus metadata
I20240502 07:59:09.786051 971 ts_tablet_manager.cc:1909] T f685643a4174404c9476127f7501370b P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:59:09.786051 973 ts_tablet_manager.cc:1888] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:09.786131 973 log.cc:1196] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-6/wal/wals/f685643a4174404c9476127f7501370b
I20240502 07:59:09.786461 973 ts_tablet_manager.cc:1909] T f685643a4174404c9476127f7501370b P 4654231dfe994c1a9d2e37d9c3d75699: Deleting consensus metadata
I20240502 07:59:09.786463 22428 catalog_manager.cc:4830] TS 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): tablet f685643a4174404c9476127f7501370b (table rebalancer_dynamic_table_new_name [id=1cd0691c88c74ef99e399d6eff035df4]) successfully deleted
I20240502 07:59:09.786832 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet f685643a4174404c9476127f7501370b (table rebalancer_dynamic_table_new_name [id=1cd0691c88c74ef99e399d6eff035df4]) successfully deleted
I20240502 07:59:09.787070 22431 catalog_manager.cc:4830] TS 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): tablet f685643a4174404c9476127f7501370b (table rebalancer_dynamic_table_new_name [id=1cd0691c88c74ef99e399d6eff035df4]) successfully deleted
W20240502 07:59:09.788075 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 07:59:09.806900 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 07:59:09.846159 505 consensus_queue.cc:1035] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:09.846374 320 consensus_queue.cc:1035] T 99597b95eb3246beb05ff4a07bd7bb84 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:09.883740 22440 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:59:09.889043 24852 tablet_service.cc:1467] Processing CreateTablet for tablet cc13a03ba1d84f0c846635e5688a3882 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=409324f322cc4d0c8d7d6a92e7c814b6]), partition=
I20240502 07:59:09.889043 22827 tablet_service.cc:1467] Processing CreateTablet for tablet cc13a03ba1d84f0c846635e5688a3882 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=409324f322cc4d0c8d7d6a92e7c814b6]), partition=
I20240502 07:59:09.889294 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cc13a03ba1d84f0c846635e5688a3882. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:09.889295 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cc13a03ba1d84f0c846635e5688a3882. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:09.889248 22957 tablet_service.cc:1467] Processing CreateTablet for tablet cc13a03ba1d84f0c846635e5688a3882 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=409324f322cc4d0c8d7d6a92e7c814b6]), partition=
I20240502 07:59:09.889492 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cc13a03ba1d84f0c846635e5688a3882. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:09.891491 980 tablet_bootstrap.cc:492] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:09.892266 979 tablet_bootstrap.cc:492] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:09.892427 980 tablet_bootstrap.cc:654] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:09.893097 981 tablet_bootstrap.cc:492] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:09.893148 979 tablet_bootstrap.cc:654] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:09.894085 979 tablet_bootstrap.cc:492] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:09.894086 980 tablet_bootstrap.cc:492] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:09.894160 980 ts_tablet_manager.cc:1367] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:59:09.894160 979 ts_tablet_manager.cc:1367] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:59:09.894536 980 raft_consensus.cc:348] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:09.894555 979 raft_consensus.cc:348] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:09.894644 980 raft_consensus.cc:374] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:09.894642 979 raft_consensus.cc:374] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:09.894677 980 raft_consensus.cc:729] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:09.894677 979 raft_consensus.cc:729] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:09.894775 979 consensus_queue.cc:260] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:09.894775 980 consensus_queue.cc:260] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:09.895057 980 ts_tablet_manager.cc:1398] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:09.895057 979 ts_tablet_manager.cc:1398] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:09.895594 981 tablet_bootstrap.cc:654] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:09.899094 981 tablet_bootstrap.cc:492] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:09.899178 981 ts_tablet_manager.cc:1367] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:59:09.899528 981 raft_consensus.cc:348] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:09.899601 981 raft_consensus.cc:374] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:09.899618 981 raft_consensus.cc:729] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:09.899714 981 consensus_queue.cc:260] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:09.899992 981 ts_tablet_manager.cc:1398] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:09.903728 406 consensus_queue.cc:1035] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:09.914703 719 consensus_queue.cc:1035] T 9caed3c626ce410da00ecdf435174d4a P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:09.986573 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:10.012248 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 07:59:10.037977 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 07:59:10.066565 438 raft_consensus.cc:482] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:10.066695 438 raft_consensus.cc:504] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
W20240502 07:59:10.066969 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 79 similar messages]
I20240502 07:59:10.067145 438 leader_election.cc:288] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:59:10.067417 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cc13a03ba1d84f0c846635e5688a3882" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:10.067551 24870 raft_consensus.cc:2454] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:59:10.067569 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cc13a03ba1d84f0c846635e5688a3882" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:10.067710 22845 raft_consensus.cc:2454] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:59:10.067816 22912 leader_election.cc:302] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:10.067965 438 raft_consensus.cc:2781] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:10.068014 438 raft_consensus.cc:482] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:10.068050 438 raft_consensus.cc:3037] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:10.069090 438 raft_consensus.cc:504] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:10.069620 438 leader_election.cc:288] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:59:10.069681 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cc13a03ba1d84f0c846635e5688a3882" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:10.069785 24870 raft_consensus.cc:3037] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:10.070331 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cc13a03ba1d84f0c846635e5688a3882" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:59:10.070475 22845 raft_consensus.cc:3037] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:10.070691 24870 raft_consensus.cc:2454] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:59:10.070956 22912 leader_election.cc:302] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:10.071118 438 raft_consensus.cc:2781] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:10.071192 438 raft_consensus.cc:686] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:59:10.071338 438 consensus_queue.cc:237] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:10.071674 22845 raft_consensus.cc:2454] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:59:10.074777 22444 catalog_manager.cc:5484] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } }
W20240502 07:59:10.104269 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 07:59:10.160881 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 07:59:10.186684 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 07:59:10.189617 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:59:10.194082 24870 raft_consensus.cc:1261] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:10.194205 22845 raft_consensus.cc:1261] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:10.194538 939 consensus_queue.cc:1035] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:10.194820 939 consensus_queue.cc:1035] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:10.197885 22444 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet cc13a03ba1d84f0c846635e5688a3882 (table rebalancer_dynamic_table [id=409324f322cc4d0c8d7d6a92e7c814b6]). Expected version 1 got 0
I20240502 07:59:10.198920 1040 mvcc.cc:205] Tried to move back new op lower bound from 7023152128789151744 to 7023152128293371904. Current Snapshot: MvccSnapshot[applied={T|T < 7023152128789151744}]
I20240502 07:59:10.199512 22438 catalog_manager.cc:5576] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459) does not have the latest schema for tablet cc13a03ba1d84f0c846635e5688a3882 (table rebalancer_dynamic_table [id=409324f322cc4d0c8d7d6a92e7c814b6]). Expected version 1 got 0
I20240502 07:59:10.200130 1049 tablet.cc:1722] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:10.200512 1049 tablet.cc:1620] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:59:10.201531 22444 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet cc13a03ba1d84f0c846635e5688a3882 (table rebalancer_dynamic_table [id=409324f322cc4d0c8d7d6a92e7c814b6]). Expected version 1 got 0
I20240502 07:59:10.201900 22438 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet cc13a03ba1d84f0c846635e5688a3882 (table rebalancer_dynamic_table [id=409324f322cc4d0c8d7d6a92e7c814b6]). Expected version 1 got 0
I20240502 07:59:10.204232 1048 tablet.cc:1722] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:10.204540 1048 tablet.cc:1620] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:10.207350 22438 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=409324f322cc4d0c8d7d6a92e7c814b6] alter complete (version 1)
I20240502 07:59:10.207685 1050 tablet.cc:1722] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:10.207926 1050 tablet.cc:1620] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
W20240502 07:59:10.273589 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:10.275704 719 consensus_queue.cc:1035] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:10.278759 719 consensus_queue.cc:1035] T e45d327e069a42088a60bc9c6f0ba797 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:10.299940 22438 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_74"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 07:59:10.301851 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20240502 07:59:10.306463 22696 tablet_service.cc:1467] Processing CreateTablet for tablet d0273d00e86a4171b09e9af3b6f90323 (DEFAULT_TABLE table=rebalancer_extra_table_74 [id=a748ca94af47401fbffc73357aa72a01]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:10.306787 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d0273d00e86a4171b09e9af3b6f90323. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:10.306810 24852 tablet_service.cc:1467] Processing CreateTablet for tablet d0273d00e86a4171b09e9af3b6f90323 (DEFAULT_TABLE table=rebalancer_extra_table_74 [id=a748ca94af47401fbffc73357aa72a01]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:10.306852 25115 tablet_service.cc:1467] Processing CreateTablet for tablet d0273d00e86a4171b09e9af3b6f90323 (DEFAULT_TABLE table=rebalancer_extra_table_74 [id=a748ca94af47401fbffc73357aa72a01]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:10.307056 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d0273d00e86a4171b09e9af3b6f90323. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:10.307118 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d0273d00e86a4171b09e9af3b6f90323. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:10.309062 980 tablet_bootstrap.cc:492] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:10.309751 1061 tablet_bootstrap.cc:492] T d0273d00e86a4171b09e9af3b6f90323 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:10.309823 22827 tablet_service.cc:1467] Processing CreateTablet for tablet a41758caaf8d4afc95b5053ec56a0485 (DEFAULT_TABLE table=rebalancer_extra_table_74 [id=a748ca94af47401fbffc73357aa72a01]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:10.310051 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a41758caaf8d4afc95b5053ec56a0485. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:10.310046 980 tablet_bootstrap.cc:654] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:10.310640 1061 tablet_bootstrap.cc:654] T d0273d00e86a4171b09e9af3b6f90323 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:10.310912 25115 tablet_service.cc:1467] Processing CreateTablet for tablet d771898dd382454783115cae94e08c55 (DEFAULT_TABLE table=rebalancer_extra_table_74 [id=a748ca94af47401fbffc73357aa72a01]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:10.311146 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d771898dd382454783115cae94e08c55. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:10.312142 981 tablet_bootstrap.cc:492] T a41758caaf8d4afc95b5053ec56a0485 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:10.312556 22696 tablet_service.cc:1467] Processing CreateTablet for tablet d771898dd382454783115cae94e08c55 (DEFAULT_TABLE table=rebalancer_extra_table_74 [id=a748ca94af47401fbffc73357aa72a01]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:10.312774 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d771898dd382454783115cae94e08c55. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:10.312916 22695 tablet_service.cc:1467] Processing CreateTablet for tablet a41758caaf8d4afc95b5053ec56a0485 (DEFAULT_TABLE table=rebalancer_extra_table_74 [id=a748ca94af47401fbffc73357aa72a01]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:10.313062 981 tablet_bootstrap.cc:654] T a41758caaf8d4afc95b5053ec56a0485 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:10.313110 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a41758caaf8d4afc95b5053ec56a0485. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:10.314744 25235 tablet_service.cc:1467] Processing CreateTablet for tablet d771898dd382454783115cae94e08c55 (DEFAULT_TABLE table=rebalancer_extra_table_74 [id=a748ca94af47401fbffc73357aa72a01]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:10.315003 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d771898dd382454783115cae94e08c55. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:10.315380 1062 tablet_bootstrap.cc:492] T d0273d00e86a4171b09e9af3b6f90323 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:10.315557 24852 tablet_service.cc:1467] Processing CreateTablet for tablet a41758caaf8d4afc95b5053ec56a0485 (DEFAULT_TABLE table=rebalancer_extra_table_74 [id=a748ca94af47401fbffc73357aa72a01]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:10.315785 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a41758caaf8d4afc95b5053ec56a0485. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:10.316296 1062 tablet_bootstrap.cc:654] T d0273d00e86a4171b09e9af3b6f90323 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:10.317143 1067 tablet_bootstrap.cc:492] T d771898dd382454783115cae94e08c55 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:10.317155 1062 tablet_bootstrap.cc:492] T d0273d00e86a4171b09e9af3b6f90323 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:10.317219 1062 ts_tablet_manager.cc:1367] T d0273d00e86a4171b09e9af3b6f90323 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:59:10.317772 1061 tablet_bootstrap.cc:492] T d0273d00e86a4171b09e9af3b6f90323 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:10.317695 1062 raft_consensus.cc:348] T d0273d00e86a4171b09e9af3b6f90323 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.317826 1062 raft_consensus.cc:374] T d0273d00e86a4171b09e9af3b6f90323 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:10.317855 1062 raft_consensus.cc:729] T d0273d00e86a4171b09e9af3b6f90323 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:10.317858 1061 ts_tablet_manager.cc:1367] T d0273d00e86a4171b09e9af3b6f90323 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.000s
I20240502 07:59:10.317966 1062 consensus_queue.cc:260] T d0273d00e86a4171b09e9af3b6f90323 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.318238 1062 ts_tablet_manager.cc:1398] T d0273d00e86a4171b09e9af3b6f90323 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:10.318292 1061 raft_consensus.cc:348] T d0273d00e86a4171b09e9af3b6f90323 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.318392 1067 tablet_bootstrap.cc:654] T d771898dd382454783115cae94e08c55 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:10.318393 1061 raft_consensus.cc:374] T d0273d00e86a4171b09e9af3b6f90323 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:10.318415 1062 tablet_bootstrap.cc:492] T d771898dd382454783115cae94e08c55 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:10.318423 1061 raft_consensus.cc:729] T d0273d00e86a4171b09e9af3b6f90323 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:10.318542 1061 consensus_queue.cc:260] T d0273d00e86a4171b09e9af3b6f90323 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.318821 1061 ts_tablet_manager.cc:1398] T d0273d00e86a4171b09e9af3b6f90323 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:10.318989 1061 tablet_bootstrap.cc:492] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:10.319212 1062 tablet_bootstrap.cc:654] T d771898dd382454783115cae94e08c55 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:10.319955 1061 tablet_bootstrap.cc:654] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:10.320904 981 tablet_bootstrap.cc:492] T a41758caaf8d4afc95b5053ec56a0485 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:10.320926 1061 tablet_bootstrap.cc:492] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:10.320982 1061 ts_tablet_manager.cc:1367] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:10.320991 981 ts_tablet_manager.cc:1367] T a41758caaf8d4afc95b5053ec56a0485 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.009s user 0.001s sys 0.000s
I20240502 07:59:10.321337 1061 raft_consensus.cc:348] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.321424 1061 raft_consensus.cc:374] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:10.321386 981 raft_consensus.cc:348] T a41758caaf8d4afc95b5053ec56a0485 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.321449 1061 raft_consensus.cc:729] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:10.321473 981 raft_consensus.cc:374] T a41758caaf8d4afc95b5053ec56a0485 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:10.321501 981 raft_consensus.cc:729] T a41758caaf8d4afc95b5053ec56a0485 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:10.321559 1061 consensus_queue.cc:260] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.321604 981 consensus_queue.cc:260] T a41758caaf8d4afc95b5053ec56a0485 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.321774 1061 ts_tablet_manager.cc:1398] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:10.321841 981 ts_tablet_manager.cc:1398] T a41758caaf8d4afc95b5053ec56a0485 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:10.322490 1062 tablet_bootstrap.cc:492] T d771898dd382454783115cae94e08c55 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:10.322592 1062 ts_tablet_manager.cc:1367] T d771898dd382454783115cae94e08c55 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.002s
I20240502 07:59:10.323561 980 tablet_bootstrap.cc:492] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:10.323649 980 ts_tablet_manager.cc:1367] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.015s user 0.002s sys 0.000s
I20240502 07:59:10.323741 1062 raft_consensus.cc:348] T d771898dd382454783115cae94e08c55 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.323832 1062 raft_consensus.cc:374] T d771898dd382454783115cae94e08c55 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:10.323860 1062 raft_consensus.cc:729] T d771898dd382454783115cae94e08c55 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:10.323945 1062 consensus_queue.cc:260] T d771898dd382454783115cae94e08c55 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.324070 980 raft_consensus.cc:348] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.324185 980 raft_consensus.cc:374] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:10.324200 1062 ts_tablet_manager.cc:1398] T d771898dd382454783115cae94e08c55 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 07:59:10.324213 980 raft_consensus.cc:729] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:10.324355 1062 tablet_bootstrap.cc:492] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:10.324326 980 consensus_queue.cc:260] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.324568 980 ts_tablet_manager.cc:1398] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:10.324753 980 tablet_bootstrap.cc:492] T a41758caaf8d4afc95b5053ec56a0485 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:10.325374 1062 tablet_bootstrap.cc:654] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:10.326462 980 tablet_bootstrap.cc:654] T a41758caaf8d4afc95b5053ec56a0485 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
W20240502 07:59:10.327178 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
I20240502 07:59:10.327616 22444 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
W20240502 07:59:10.328472 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 07:59:10.329428 980 tablet_bootstrap.cc:492] T a41758caaf8d4afc95b5053ec56a0485 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:10.329504 980 ts_tablet_manager.cc:1367] T a41758caaf8d4afc95b5053ec56a0485 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.002s
I20240502 07:59:10.329878 980 raft_consensus.cc:348] T a41758caaf8d4afc95b5053ec56a0485 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.329955 980 raft_consensus.cc:374] T a41758caaf8d4afc95b5053ec56a0485 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:10.329980 980 raft_consensus.cc:729] T a41758caaf8d4afc95b5053ec56a0485 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:10.330067 980 consensus_queue.cc:260] T a41758caaf8d4afc95b5053ec56a0485 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.330319 980 ts_tablet_manager.cc:1398] T a41758caaf8d4afc95b5053ec56a0485 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:10.335605 1062 tablet_bootstrap.cc:492] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:10.335731 1062 ts_tablet_manager.cc:1367] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.011s user 0.001s sys 0.000s
I20240502 07:59:10.335927 1049 tablet.cc:1722] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:10.336251 1049 tablet.cc:1620] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:59:10.336252 1048 tablet.cc:1722] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:10.336239 1062 raft_consensus.cc:348] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.336337 1062 raft_consensus.cc:374] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:10.336362 1062 raft_consensus.cc:729] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:10.336572 1048 tablet.cc:1620] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:10.336555 1062 consensus_queue.cc:260] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.336891 1062 ts_tablet_manager.cc:1398] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:10.337934 643 raft_consensus.cc:482] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:10.338006 643 raft_consensus.cc:504] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.338415 643 leader_election.cc:288] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:59:10.339288 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a41758caaf8d4afc95b5053ec56a0485" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:10.339413 24871 raft_consensus.cc:2454] T a41758caaf8d4afc95b5053ec56a0485 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:59:10.340471 22650 leader_election.cc:302] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:10.340612 643 raft_consensus.cc:2781] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:10.340683 643 raft_consensus.cc:482] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:10.340714 643 raft_consensus.cc:3037] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:10.341078 320 raft_consensus.cc:482] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:10.341158 320 raft_consensus.cc:504] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.341471 320 leader_election.cc:288] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:10.341714 643 raft_consensus.cc:504] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.341953 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d771898dd382454783115cae94e08c55" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:10.342036 643 leader_election.cc:288] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:59:10.342054 22717 raft_consensus.cc:2454] T d771898dd382454783115cae94e08c55 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 0.
I20240502 07:59:10.342564 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a41758caaf8d4afc95b5053ec56a0485" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:59:10.342662 22845 raft_consensus.cc:3037] T a41758caaf8d4afc95b5053ec56a0485 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:10.343508 22845 raft_consensus.cc:2454] T a41758caaf8d4afc95b5053ec56a0485 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:59:10.343740 22653 leader_election.cc:302] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:10.344066 1050 tablet.cc:1722] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:10.344319 1050 tablet.cc:1620] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537: MemRowSet was empty: no flush needed.
I20240502 07:59:10.344833 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d771898dd382454783115cae94e08c55" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:10.345553 25068 leader_election.cc:302] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:10.345597 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a41758caaf8d4afc95b5053ec56a0485" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:10.345701 24870 raft_consensus.cc:3037] T a41758caaf8d4afc95b5053ec56a0485 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:10.345853 320 raft_consensus.cc:2781] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:10.345917 320 raft_consensus.cc:482] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:10.345955 320 raft_consensus.cc:3037] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
W20240502 07:59:10.346052 25070 leader_election.cc:341] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Illegal state: must be running to vote when last-logged opid is not known
I20240502 07:59:10.346045 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a41758caaf8d4afc95b5053ec56a0485" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:10.346190 22847 raft_consensus.cc:2362] T a41758caaf8d4afc95b5053ec56a0485 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1. Re-sending same reply.
I20240502 07:59:10.346664 24870 raft_consensus.cc:2454] T a41758caaf8d4afc95b5053ec56a0485 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:59:10.346966 320 raft_consensus.cc:504] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.347206 1067 tablet_bootstrap.cc:492] T d771898dd382454783115cae94e08c55 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:10.347293 1067 ts_tablet_manager.cc:1367] T d771898dd382454783115cae94e08c55 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.030s user 0.002s sys 0.000s
I20240502 07:59:10.347321 320 leader_election.cc:288] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:10.347537 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d771898dd382454783115cae94e08c55" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:10.347822 1067 raft_consensus.cc:348] T d771898dd382454783115cae94e08c55 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.348078 1067 raft_consensus.cc:374] T d771898dd382454783115cae94e08c55 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:10.348131 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=409324f322cc4d0c8d7d6a92e7c814b6] alter complete (version 2)
I20240502 07:59:10.348160 1067 raft_consensus.cc:729] T d771898dd382454783115cae94e08c55 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:10.348306 1067 consensus_queue.cc:260] T d771898dd382454783115cae94e08c55 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.348579 643 raft_consensus.cc:2781] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:10.348639 1067 ts_tablet_manager.cc:1398] T d771898dd382454783115cae94e08c55 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:10.348703 643 raft_consensus.cc:686] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:59:10.348848 643 consensus_queue.cc:237] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.348930 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d771898dd382454783115cae94e08c55" candidate_uuid: "4654231dfe994c1a9d2e37d9c3d75699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:10.349028 22717 raft_consensus.cc:3037] T d771898dd382454783115cae94e08c55 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:10.349887 22717 raft_consensus.cc:2454] T d771898dd382454783115cae94e08c55 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4654231dfe994c1a9d2e37d9c3d75699 in term 1.
W20240502 07:59:10.350334 25070 leader_election.cc:341] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769): Illegal state: must be running to vote when last-logged opid is not known
I20240502 07:59:10.350634 25068 leader_election.cc:302] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 725969d490a04fdfb3ba890c3f5096e3; no voters: 7eced220cf1642f3873fc385b6191e69
I20240502 07:59:10.352299 22436 catalog_manager.cc:5484] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } }
I20240502 07:59:10.352744 320 raft_consensus.cc:2781] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:10.352818 320 raft_consensus.cc:686] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Becoming Leader. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Running, Role: LEADER
I20240502 07:59:10.352938 320 consensus_queue.cc:237] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.355399 22444 catalog_manager.cc:5484] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135). New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
W20240502 07:59:10.367679 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 07:59:10.427738 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 07:59:10.428402 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:10.441148 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 07:59:10.441726 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 07:59:10.455386 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:59:10.461977 1048 tablet.cc:1722] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:10.462297 1049 tablet.cc:1722] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:10.463289 1050 tablet.cc:1722] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:10.467756 22444 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=409324f322cc4d0c8d7d6a92e7c814b6] alter complete (version 3)
W20240502 07:59:10.472301 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 07:59:10.534696 846 raft_consensus.cc:482] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:10.534812 846 raft_consensus.cc:504] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.535337 846 leader_election.cc:288] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:10.535682 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d0273d00e86a4171b09e9af3b6f90323" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:10.535686 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d0273d00e86a4171b09e9af3b6f90323" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:10.535815 22715 raft_consensus.cc:2454] T d0273d00e86a4171b09e9af3b6f90323 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:10.535832 25135 raft_consensus.cc:2454] T d0273d00e86a4171b09e9af3b6f90323 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 0.
I20240502 07:59:10.536069 24805 leader_election.cc:302] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:10.536302 846 raft_consensus.cc:2781] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:10.536346 846 raft_consensus.cc:482] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:10.536391 846 raft_consensus.cc:3037] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:10.537268 846 raft_consensus.cc:504] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.537539 846 leader_election.cc:288] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Requested vote from peers 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205), 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 07:59:10.537807 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d0273d00e86a4171b09e9af3b6f90323" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:10.537856 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d0273d00e86a4171b09e9af3b6f90323" candidate_uuid: "6d02017318a2461796d431962fe6a537" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:10.537928 25135 raft_consensus.cc:3037] T d0273d00e86a4171b09e9af3b6f90323 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:10.537972 22715 raft_consensus.cc:3037] T d0273d00e86a4171b09e9af3b6f90323 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:10.538898 25135 raft_consensus.cc:2454] T d0273d00e86a4171b09e9af3b6f90323 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:10.538969 22715 raft_consensus.cc:2454] T d0273d00e86a4171b09e9af3b6f90323 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d02017318a2461796d431962fe6a537 in term 1.
I20240502 07:59:10.539163 24808 leader_election.cc:302] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 6d02017318a2461796d431962fe6a537; no voters:
I20240502 07:59:10.539315 846 raft_consensus.cc:2781] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:10.539484 846 raft_consensus.cc:686] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Becoming Leader. State: Replica: 6d02017318a2461796d431962fe6a537, State: Running, Role: LEADER
I20240502 07:59:10.539634 846 consensus_queue.cc:237] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 07:59:10.541893 22436 catalog_manager.cc:5484] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6d02017318a2461796d431962fe6a537 (127.16.85.133). New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:10.569913 22436 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:10.570050 22436 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:10.572032 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet cc13a03ba1d84f0c846635e5688a3882
I20240502 07:59:10.572571 24852 tablet_service.cc:1514] Processing DeleteTablet for tablet cc13a03ba1d84f0c846635e5688a3882 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:10 UTC) from {username='slave'} at 127.0.0.1:33546
I20240502 07:59:10.572611 22827 tablet_service.cc:1514] Processing DeleteTablet for tablet cc13a03ba1d84f0c846635e5688a3882 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:10 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:59:10.573035 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet cc13a03ba1d84f0c846635e5688a3882 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:10 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:59:10.573170 1110 tablet_replica.cc:330] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:59:10.573308 1110 raft_consensus.cc:2229] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:10.573329 1111 tablet_replica.cc:330] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537: stopping tablet replica
I20240502 07:59:10.573397 1112 tablet_replica.cc:330] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:59:10.573519 1111 raft_consensus.cc:2229] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:10.573529 1110 raft_consensus.cc:2258] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:10.573582 1112 raft_consensus.cc:2229] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:59:10.573685 1111 raft_consensus.cc:2258] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:10.574046 1112 raft_consensus.cc:2258] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:10.574405 1110 ts_tablet_manager.cc:1875] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:10.574438 1111 ts_tablet_manager.cc:1875] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:10.574623 1112 ts_tablet_manager.cc:1875] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:10.576680 1110 ts_tablet_manager.cc:1888] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:10.576792 1110 log.cc:1196] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/cc13a03ba1d84f0c846635e5688a3882
I20240502 07:59:10.576931 1111 ts_tablet_manager.cc:1888] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:10.577046 1111 log.cc:1196] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-4/wal/wals/cc13a03ba1d84f0c846635e5688a3882
I20240502 07:59:10.577047 1112 ts_tablet_manager.cc:1888] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:10.577116 1112 log.cc:1196] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/cc13a03ba1d84f0c846635e5688a3882
I20240502 07:59:10.577402 1110 ts_tablet_manager.cc:1909] T cc13a03ba1d84f0c846635e5688a3882 P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:59:10.577472 1111 ts_tablet_manager.cc:1909] T cc13a03ba1d84f0c846635e5688a3882 P 6d02017318a2461796d431962fe6a537: Deleting consensus metadata
I20240502 07:59:10.577509 1112 ts_tablet_manager.cc:1909] T cc13a03ba1d84f0c846635e5688a3882 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:59:10.578058 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet cc13a03ba1d84f0c846635e5688a3882 (table rebalancer_dynamic_table_new_name [id=409324f322cc4d0c8d7d6a92e7c814b6]) successfully deleted
I20240502 07:59:10.578096 22428 catalog_manager.cc:4830] TS 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): tablet cc13a03ba1d84f0c846635e5688a3882 (table rebalancer_dynamic_table_new_name [id=409324f322cc4d0c8d7d6a92e7c814b6]) successfully deleted
I20240502 07:59:10.578192 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet cc13a03ba1d84f0c846635e5688a3882 (table rebalancer_dynamic_table_new_name [id=409324f322cc4d0c8d7d6a92e7c814b6]) successfully deleted
W20240502 07:59:10.602152 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 07:59:10.602798 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 07:59:10.630681 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 07:59:10.673607 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:59:10.679306 22827 tablet_service.cc:1467] Processing CreateTablet for tablet 5923b17ed0474cd0a4f7763ce9ea5734 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=30c5e8c428744d3cad4c1f5cd04104bb]), partition=
I20240502 07:59:10.679380 22957 tablet_service.cc:1467] Processing CreateTablet for tablet 5923b17ed0474cd0a4f7763ce9ea5734 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=30c5e8c428744d3cad4c1f5cd04104bb]), partition=
I20240502 07:59:10.679474 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 5923b17ed0474cd0a4f7763ce9ea5734 (DEFAULT_TABLE table=rebalancer_dynamic_table [id=30c5e8c428744d3cad4c1f5cd04104bb]), partition=
I20240502 07:59:10.679586 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5923b17ed0474cd0a4f7763ce9ea5734. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:10.679586 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5923b17ed0474cd0a4f7763ce9ea5734. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:10.679787 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5923b17ed0474cd0a4f7763ce9ea5734. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:10.681818 1127 tablet_bootstrap.cc:492] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:10.682109 981 tablet_bootstrap.cc:492] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:10.682704 1127 tablet_bootstrap.cc:654] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:10.683036 981 tablet_bootstrap.cc:654] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:10.683588 1128 tablet_bootstrap.cc:492] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:10.684384 1128 tablet_bootstrap.cc:654] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:10.684944 1127 tablet_bootstrap.cc:492] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:10.685051 1127 ts_tablet_manager.cc:1367] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:59:10.685624 1127 raft_consensus.cc:348] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:10.685729 1128 tablet_bootstrap.cc:492] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:10.685755 1127 raft_consensus.cc:374] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:10.685787 1127 raft_consensus.cc:729] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:10.685856 1128 ts_tablet_manager.cc:1367] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:59:10.685942 1127 consensus_queue.cc:260] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:10.686291 1128 raft_consensus.cc:348] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:10.686403 1128 raft_consensus.cc:374] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:10.686443 1128 raft_consensus.cc:729] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:10.686561 1128 consensus_queue.cc:260] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:10.686673 1127 ts_tablet_manager.cc:1398] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:10.686903 1128 ts_tablet_manager.cc:1398] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:10.687158 981 tablet_bootstrap.cc:492] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:10.687242 981 ts_tablet_manager.cc:1367] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.001s
I20240502 07:59:10.687624 981 raft_consensus.cc:348] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:10.687716 981 raft_consensus.cc:374] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:10.687741 981 raft_consensus.cc:729] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:10.687852 981 consensus_queue.cc:260] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:10.688100 981 ts_tablet_manager.cc:1398] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:10.735743 643 consensus_queue.cc:1035] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:10.738119 25265 raft_consensus.cc:3037] T d771898dd382454783115cae94e08c55 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:10.739647 320 consensus_queue.cc:1035] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:10.744225 982 consensus_queue.cc:1035] T a41758caaf8d4afc95b5053ec56a0485 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:10.749042 976 consensus_queue.cc:1035] T d771898dd382454783115cae94e08c55 P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:10.838625 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:10.910113 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 07:59:10.926971 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
I20240502 07:59:10.942867 719 raft_consensus.cc:482] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:10.942970 719 raft_consensus.cc:504] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:10.943404 719 leader_election.cc:288] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:10.943797 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5923b17ed0474cd0a4f7763ce9ea5734" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:10.943935 22978 raft_consensus.cc:2454] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:10.943969 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5923b17ed0474cd0a4f7763ce9ea5734" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:10.944100 22845 raft_consensus.cc:2454] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 0.
I20240502 07:59:10.944416 22522 leader_election.cc:302] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:10.945554 719 raft_consensus.cc:2781] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:10.945616 719 raft_consensus.cc:482] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:10.945654 719 raft_consensus.cc:3037] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:10.946831 719 raft_consensus.cc:504] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:10.947197 719 leader_election.cc:288] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:10.947599 22845 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5923b17ed0474cd0a4f7763ce9ea5734" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:59:10.947713 22845 raft_consensus.cc:3037] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:10.947932 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5923b17ed0474cd0a4f7763ce9ea5734" candidate_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:10.948048 22978 raft_consensus.cc:3037] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:10.948519 22845 raft_consensus.cc:2454] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:10.948940 22522 leader_election.cc:302] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:10.949085 719 raft_consensus.cc:2781] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:10.949091 22978 raft_consensus.cc:2454] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bea0fe8d8044a2685f92f7a10c9cf85 in term 1.
I20240502 07:59:10.949148 719 raft_consensus.cc:686] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Becoming Leader. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Running, Role: LEADER
I20240502 07:59:10.949259 719 consensus_queue.cc:237] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:10.951593 22440 catalog_manager.cc:5484] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129). New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } }
W20240502 07:59:11.005815 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 07:59:11.017699 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 07:59:11.040817 846 consensus_queue.cc:1035] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:11.043444 1187 consensus_queue.cc:1035] T d0273d00e86a4171b09e9af3b6f90323 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:11.072566 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_76"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:59:11.078898 22957 tablet_service.cc:1467] Processing CreateTablet for tablet e036a6ba0f704f86a13e81792235e7d6 (DEFAULT_TABLE table=rebalancer_extra_table_76 [id=77b98dc7818d4f1ca1125a07fcfa70c0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.079262 22957 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e036a6ba0f704f86a13e81792235e7d6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.080327 25113 tablet_service.cc:1467] Processing CreateTablet for tablet 5b6b4c02a8a749d2afe291578af32a4b (DEFAULT_TABLE table=rebalancer_extra_table_76 [id=77b98dc7818d4f1ca1125a07fcfa70c0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.080327 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 5b6b4c02a8a749d2afe291578af32a4b (DEFAULT_TABLE table=rebalancer_extra_table_76 [id=77b98dc7818d4f1ca1125a07fcfa70c0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.080605 25113 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5b6b4c02a8a749d2afe291578af32a4b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.080605 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5b6b4c02a8a749d2afe291578af32a4b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.080883 25115 tablet_service.cc:1467] Processing CreateTablet for tablet 4a457bdbe18d4c779f93c32c7c59d3a5 (DEFAULT_TABLE table=rebalancer_extra_table_76 [id=77b98dc7818d4f1ca1125a07fcfa70c0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.081096 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4a457bdbe18d4c779f93c32c7c59d3a5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.082588 1128 tablet_bootstrap.cc:492] T 5b6b4c02a8a749d2afe291578af32a4b P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:11.082780 24852 tablet_service.cc:1467] Processing CreateTablet for tablet e036a6ba0f704f86a13e81792235e7d6 (DEFAULT_TABLE table=rebalancer_extra_table_76 [id=77b98dc7818d4f1ca1125a07fcfa70c0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.083039 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e036a6ba0f704f86a13e81792235e7d6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.083531 1128 tablet_bootstrap.cc:654] T 5b6b4c02a8a749d2afe291578af32a4b P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.084169 1189 tablet_bootstrap.cc:492] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:11.084507 1128 tablet_bootstrap.cc:492] T 5b6b4c02a8a749d2afe291578af32a4b P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:11.084587 1128 ts_tablet_manager.cc:1367] T 5b6b4c02a8a749d2afe291578af32a4b P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.002s
I20240502 07:59:11.085084 1189 tablet_bootstrap.cc:654] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.085042 1128 raft_consensus.cc:348] T 5b6b4c02a8a749d2afe291578af32a4b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:11.085140 1128 raft_consensus.cc:374] T 5b6b4c02a8a749d2afe291578af32a4b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.085170 1128 raft_consensus.cc:729] T 5b6b4c02a8a749d2afe291578af32a4b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.085286 1128 consensus_queue.cc:260] T 5b6b4c02a8a749d2afe291578af32a4b P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:11.085515 1128 ts_tablet_manager.cc:1398] T 5b6b4c02a8a749d2afe291578af32a4b P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:11.085521 1191 tablet_bootstrap.cc:492] T e036a6ba0f704f86a13e81792235e7d6 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:11.086485 1191 tablet_bootstrap.cc:654] T e036a6ba0f704f86a13e81792235e7d6 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.087098 1189 tablet_bootstrap.cc:492] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:11.087246 1189 ts_tablet_manager.cc:1367] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240502 07:59:11.087489 1191 tablet_bootstrap.cc:492] T e036a6ba0f704f86a13e81792235e7d6 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:11.087653 1191 ts_tablet_manager.cc:1367] T e036a6ba0f704f86a13e81792235e7d6 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:11.087702 1189 raft_consensus.cc:348] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:11.087860 1189 raft_consensus.cc:374] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.087929 1189 raft_consensus.cc:729] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.088080 1189 consensus_queue.cc:260] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:11.088157 1191 raft_consensus.cc:348] T e036a6ba0f704f86a13e81792235e7d6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.088260 1191 raft_consensus.cc:374] T e036a6ba0f704f86a13e81792235e7d6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.088300 1191 raft_consensus.cc:729] T e036a6ba0f704f86a13e81792235e7d6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.088408 1191 consensus_queue.cc:260] T e036a6ba0f704f86a13e81792235e7d6 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.088752 1191 ts_tablet_manager.cc:1398] T e036a6ba0f704f86a13e81792235e7d6 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:11.089231 22440 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: DROP_COLUMN drop_column { name: "a" } } modify_external_catalogs: true
I20240502 07:59:11.091226 976 raft_consensus.cc:482] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:11.091300 976 raft_consensus.cc:504] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:11.091662 976 leader_election.cc:288] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:11.092799 22827 tablet_service.cc:1467] Processing CreateTablet for tablet e036a6ba0f704f86a13e81792235e7d6 (DEFAULT_TABLE table=rebalancer_extra_table_76 [id=77b98dc7818d4f1ca1125a07fcfa70c0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.092873 22826 tablet_service.cc:1467] Processing CreateTablet for tablet 4a457bdbe18d4c779f93c32c7c59d3a5 (DEFAULT_TABLE table=rebalancer_extra_table_76 [id=77b98dc7818d4f1ca1125a07fcfa70c0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.093067 22827 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e036a6ba0f704f86a13e81792235e7d6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.093046 22958 tablet_service.cc:1467] Processing CreateTablet for tablet 5b6b4c02a8a749d2afe291578af32a4b (DEFAULT_TABLE table=rebalancer_extra_table_76 [id=77b98dc7818d4f1ca1125a07fcfa70c0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.093289 22958 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5b6b4c02a8a749d2afe291578af32a4b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.093319 22959 tablet_service.cc:1467] Processing CreateTablet for tablet 4a457bdbe18d4c779f93c32c7c59d3a5 (DEFAULT_TABLE table=rebalancer_extra_table_76 [id=77b98dc7818d4f1ca1125a07fcfa70c0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
W20240502 07:59:11.093429 25069 leader_election.cc:341] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): Not found: Tablet not found: 4a457bdbe18d4c779f93c32c7c59d3a5
I20240502 07:59:11.093531 22959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4a457bdbe18d4c779f93c32c7c59d3a5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.094019 1189 ts_tablet_manager.cc:1398] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.007s user 0.000s sys 0.001s
I20240502 07:59:11.095172 981 tablet_bootstrap.cc:492] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:11.095219 1127 tablet_bootstrap.cc:492] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:11.096086 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4a457bdbe18d4c779f93c32c7c59d3a5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.096133 981 tablet_bootstrap.cc:654] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.096148 1127 tablet_bootstrap.cc:654] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.098120 1189 tablet_bootstrap.cc:492] T 5b6b4c02a8a749d2afe291578af32a4b P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:11.099099 1189 tablet_bootstrap.cc:654] T 5b6b4c02a8a749d2afe291578af32a4b P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.100104 981 tablet_bootstrap.cc:492] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:11.100186 981 ts_tablet_manager.cc:1367] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.001s
W20240502 07:59:11.100231 25071 leader_election.cc:341] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): Not found: Tablet not found: 4a457bdbe18d4c779f93c32c7c59d3a5
I20240502 07:59:11.100301 25071 leader_election.cc:302] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699; no voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88
I20240502 07:59:11.100497 976 raft_consensus.cc:2726] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 07:59:11.100638 981 raft_consensus.cc:348] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.100757 981 raft_consensus.cc:374] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.100787 981 raft_consensus.cc:729] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.100898 981 consensus_queue.cc:260] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.101146 981 ts_tablet_manager.cc:1398] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:11.101740 981 tablet_bootstrap.cc:492] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:11.102454 1127 tablet_bootstrap.cc:492] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:11.102558 1127 ts_tablet_manager.cc:1367] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.000s
I20240502 07:59:11.102622 981 tablet_bootstrap.cc:654] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.102941 1127 raft_consensus.cc:348] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:11.103061 1127 raft_consensus.cc:374] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.103091 1127 raft_consensus.cc:729] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.103201 1127 consensus_queue.cc:260] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:11.103504 1127 ts_tablet_manager.cc:1398] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:11.103662 1127 tablet_bootstrap.cc:492] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:11.104509 1127 tablet_bootstrap.cc:654] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.104552 981 tablet_bootstrap.cc:492] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:11.104614 981 ts_tablet_manager.cc:1367] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.001s
I20240502 07:59:11.105540 1127 tablet_bootstrap.cc:492] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:11.105610 1127 ts_tablet_manager.cc:1367] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:11.105758 981 raft_consensus.cc:348] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:11.106038 1127 raft_consensus.cc:348] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:11.106163 1127 raft_consensus.cc:374] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.106215 1127 raft_consensus.cc:729] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.106333 1127 consensus_queue.cc:260] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:11.106824 1127 ts_tablet_manager.cc:1398] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:11.106973 1127 tablet_bootstrap.cc:492] T e036a6ba0f704f86a13e81792235e7d6 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 07:59:11.107779 1127 tablet_bootstrap.cc:654] T e036a6ba0f704f86a13e81792235e7d6 P 02a92e1c084b4308b9e855a4cc2c295f: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.107896 981 raft_consensus.cc:374] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.107945 981 raft_consensus.cc:729] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.108050 981 consensus_queue.cc:260] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:11.108609 22978 raft_consensus.cc:1261] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:11.109041 1057 consensus_queue.cc:1035] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:11.109246 22845 raft_consensus.cc:1261] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 07:59:11.109292 1127 tablet_bootstrap.cc:492] T e036a6ba0f704f86a13e81792235e7d6 P 02a92e1c084b4308b9e855a4cc2c295f: No bootstrap required, opened a new log
I20240502 07:59:11.109366 1127 ts_tablet_manager.cc:1367] T e036a6ba0f704f86a13e81792235e7d6 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:11.109516 1057 consensus_queue.cc:1035] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:11.109740 1127 raft_consensus.cc:348] T e036a6ba0f704f86a13e81792235e7d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.109834 1127 raft_consensus.cc:374] T e036a6ba0f704f86a13e81792235e7d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.109865 1127 raft_consensus.cc:729] T e036a6ba0f704f86a13e81792235e7d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.109966 1127 consensus_queue.cc:260] T e036a6ba0f704f86a13e81792235e7d6 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.110257 1127 ts_tablet_manager.cc:1398] T e036a6ba0f704f86a13e81792235e7d6 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:11.112982 981 ts_tablet_manager.cc:1398] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.008s user 0.001s sys 0.000s
I20240502 07:59:11.113451 22436 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet 5923b17ed0474cd0a4f7763ce9ea5734 (table rebalancer_dynamic_table [id=30c5e8c428744d3cad4c1f5cd04104bb]). Expected version 1 got 0
I20240502 07:59:11.113658 1189 tablet_bootstrap.cc:492] T 5b6b4c02a8a749d2afe291578af32a4b P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:11.113754 1189 ts_tablet_manager.cc:1367] T 5b6b4c02a8a749d2afe291578af32a4b P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.019s user 0.000s sys 0.002s
I20240502 07:59:11.114120 438 raft_consensus.cc:482] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:11.114188 438 raft_consensus.cc:504] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:11.114326 1189 raft_consensus.cc:348] T 5b6b4c02a8a749d2afe291578af32a4b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:11.114423 1189 raft_consensus.cc:374] T 5b6b4c02a8a749d2afe291578af32a4b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.114459 1189 raft_consensus.cc:729] T 5b6b4c02a8a749d2afe291578af32a4b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.114493 438 leader_election.cc:288] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:11.114609 1189 consensus_queue.cc:260] T 5b6b4c02a8a749d2afe291578af32a4b P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:11.114882 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5b6b4c02a8a749d2afe291578af32a4b" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:11.114881 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5b6b4c02a8a749d2afe291578af32a4b" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:11.114992 25135 raft_consensus.cc:2454] T 5b6b4c02a8a749d2afe291578af32a4b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:59:11.115039 22584 raft_consensus.cc:2454] T 5b6b4c02a8a749d2afe291578af32a4b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 0.
I20240502 07:59:11.115080 1189 ts_tablet_manager.cc:1398] T 5b6b4c02a8a749d2afe291578af32a4b P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:11.115283 22915 leader_election.cc:302] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:11.115736 438 raft_consensus.cc:2781] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:11.115790 438 raft_consensus.cc:482] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:11.115830 438 raft_consensus.cc:3037] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:11.116809 438 raft_consensus.cc:504] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:11.117100 438 leader_election.cc:288] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Requested vote from peers 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:11.117635 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5b6b4c02a8a749d2afe291578af32a4b" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:11.117729 25135 raft_consensus.cc:3037] T 5b6b4c02a8a749d2afe291578af32a4b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:11.118685 25135 raft_consensus.cc:2454] T 5b6b4c02a8a749d2afe291578af32a4b P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
I20240502 07:59:11.118705 988 mvcc.cc:205] Tried to move back new op lower bound from 7023152132535767040 to 7023152131889455104. Current Snapshot: MvccSnapshot[applied={T|T < 7023152132535767040}]
I20240502 07:59:11.119086 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5b6b4c02a8a749d2afe291578af32a4b" candidate_uuid: "02a92e1c084b4308b9e855a4cc2c295f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:11.119182 22584 raft_consensus.cc:3037] T 5b6b4c02a8a749d2afe291578af32a4b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:11.120091 22584 raft_consensus.cc:2454] T 5b6b4c02a8a749d2afe291578af32a4b P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 02a92e1c084b4308b9e855a4cc2c295f in term 1.
W20240502 07:59:11.120852 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 07:59:11.121704 22444 catalog_manager.cc:5576] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425) does not have the latest schema for tablet 5923b17ed0474cd0a4f7763ce9ea5734 (table rebalancer_dynamic_table [id=30c5e8c428744d3cad4c1f5cd04104bb]). Expected version 1 got 0
I20240502 07:59:11.121753 22915 leader_election.cc:302] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 4654231dfe994c1a9d2e37d9c3d75699; no voters:
I20240502 07:59:11.121908 1107 raft_consensus.cc:2781] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:11.121975 1107 raft_consensus.cc:686] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Becoming Leader. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Running, Role: LEADER
I20240502 07:59:11.122087 1107 consensus_queue.cc:237] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 07:59:11.123930 1152 raft_consensus.cc:482] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:11.124023 1152 raft_consensus.cc:504] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:11.124402 1152 leader_election.cc:288] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:11.124754 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4a457bdbe18d4c779f93c32c7c59d3a5" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:11.124846 22978 raft_consensus.cc:2454] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:11.125097 22782 leader_election.cc:302] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:11.125288 1152 raft_consensus.cc:2781] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:11.125341 1152 raft_consensus.cc:482] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:11.125376 1152 raft_consensus.cc:3037] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:11.125531 22444 catalog_manager.cc:5576] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055) does not have the latest schema for tablet 5923b17ed0474cd0a4f7763ce9ea5734 (table rebalancer_dynamic_table [id=30c5e8c428744d3cad4c1f5cd04104bb]). Expected version 1 got 0
I20240502 07:59:11.125584 22438 catalog_manager.cc:5484] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: term changed from 0 to 1, leader changed from <none> to 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132). New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } }
I20240502 07:59:11.125926 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4a457bdbe18d4c779f93c32c7c59d3a5" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:11.126073 25134 raft_consensus.cc:2454] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:11.126487 1152 raft_consensus.cc:504] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:11.126821 1152 leader_election.cc:288] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:11.127043 22978 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4a457bdbe18d4c779f93c32c7c59d3a5" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:11.127133 22978 raft_consensus.cc:3037] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:11.127374 22440 catalog_manager.cc:5576] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451) does not have the latest schema for tablet 5923b17ed0474cd0a4f7763ce9ea5734 (table rebalancer_dynamic_table [id=30c5e8c428744d3cad4c1f5cd04104bb]). Expected version 1 got 0
I20240502 07:59:11.127620 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4a457bdbe18d4c779f93c32c7c59d3a5" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:11.127717 25134 raft_consensus.cc:3037] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:11.128082 22978 raft_consensus.cc:2454] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:11.128350 22782 leader_election.cc:302] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:11.128505 466 raft_consensus.cc:2781] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:11.128571 466 raft_consensus.cc:686] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:59:11.128719 25134 raft_consensus.cc:2454] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:11.128793 466 consensus_queue.cc:237] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } }
I20240502 07:59:11.129463 1204 tablet.cc:1722] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:11.129845 1204 tablet.cc:1620] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:59:11.130271 1206 tablet.cc:1722] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:11.130569 1206 tablet.cc:1620] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:59:11.130957 1205 tablet.cc:1722] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
11:a INT64 NULLABLE,
PRIMARY KEY (key)
) version 0 to (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1
I20240502 07:59:11.131970 1205 tablet.cc:1620] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:11.134739 22438 catalog_manager.cc:5484] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:11.136076 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=30c5e8c428744d3cad4c1f5cd04104bb] alter complete (version 1)
W20240502 07:59:11.150295 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 07:59:11.226336 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 07:59:11.227411 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 07:59:11.238131 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } alter_schema_steps { type: ADD_COLUMN add_column { schema { name: "a" type: STRING is_nullable: true encoding: AUTO_ENCODING compression: DEFAULT_COMPRESSION cfile_block_size: 0 immutable: false } } } modify_external_catalogs: true
I20240502 07:59:11.243234 1204 tablet.cc:1722] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:11.243614 1204 tablet.cc:1620] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85: MemRowSet was empty: no flush needed.
I20240502 07:59:11.243597 1205 tablet.cc:1722] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:11.243856 1205 tablet.cc:1620] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f: MemRowSet was empty: no flush needed.
I20240502 07:59:11.244410 1206 tablet.cc:1722] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
PRIMARY KEY (key)
) version 1 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2
I20240502 07:59:11.244706 1206 tablet.cc:1620] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88: MemRowSet was empty: no flush needed.
I20240502 07:59:11.250533 22436 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table [id=30c5e8c428744d3cad4c1f5cd04104bb] alter complete (version 2)
W20240502 07:59:11.279788 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 07:59:11.312225 466 raft_consensus.cc:482] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:11.312357 466 raft_consensus.cc:504] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.312769 466 leader_election.cc:288] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:59:11.313215 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e036a6ba0f704f86a13e81792235e7d6" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f" is_pre_election: true
I20240502 07:59:11.313222 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e036a6ba0f704f86a13e81792235e7d6" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:11.313326 22979 raft_consensus.cc:2454] T e036a6ba0f704f86a13e81792235e7d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:11.313357 24871 raft_consensus.cc:2454] T e036a6ba0f704f86a13e81792235e7d6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:11.313532 22782 leader_election.cc:302] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:11.313721 466 raft_consensus.cc:2781] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:11.313764 466 raft_consensus.cc:482] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:11.313817 466 raft_consensus.cc:3037] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:11.314640 466 raft_consensus.cc:504] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.314908 466 leader_election.cc:288] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:59:11.315203 24871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e036a6ba0f704f86a13e81792235e7d6" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:11.315222 22979 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e036a6ba0f704f86a13e81792235e7d6" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02a92e1c084b4308b9e855a4cc2c295f"
I20240502 07:59:11.315323 24871 raft_consensus.cc:3037] T e036a6ba0f704f86a13e81792235e7d6 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:11.315323 22979 raft_consensus.cc:3037] T e036a6ba0f704f86a13e81792235e7d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:11.316277 24871 raft_consensus.cc:2454] T e036a6ba0f704f86a13e81792235e7d6 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:11.316277 22979 raft_consensus.cc:2454] T e036a6ba0f704f86a13e81792235e7d6 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:11.316573 22782 leader_election.cc:302] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02a92e1c084b4308b9e855a4cc2c295f, 7db4178eed174225bfb24f5f9445ce88; no voters:
I20240502 07:59:11.316766 466 raft_consensus.cc:2781] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:11.316972 466 raft_consensus.cc:686] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:59:11.317097 466 consensus_queue.cc:237] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.319344 22436 catalog_manager.cc:5484] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:11.351886 22436 catalog_manager.cc:3330] Servicing AlterTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table" } new_table_name: "rebalancer_dynamic_table_new_name" modify_external_catalogs: true
I20240502 07:59:11.357054 1204 tablet.cc:1722] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:11.357736 1205 tablet.cc:1722] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:11.358105 1206 tablet.cc:1722] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88: Alter schema from (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 2 to (
10:key INT64 NOT NULL,
12:a STRING NULLABLE,
PRIMARY KEY (key)
) version 3
I20240502 07:59:11.364050 22440 catalog_manager.cc:6029] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: rebalancer_dynamic_table_new_name [id=30c5e8c428744d3cad4c1f5cd04104bb] alter complete (version 3)
W20240502 07:59:11.432535 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:11.476789 22436 catalog_manager.cc:2458] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:11.476938 22436 catalog_manager.cc:2642] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:42052:
table { table_name: "rebalancer_dynamic_table_new_name" } modify_external_catalogs: true
I20240502 07:59:11.478799 22436 catalog_manager.cc:5771] T 00000000000000000000000000000000 P a69026f7179945c0b6a4b19bfb88d16e: Sending DeleteTablet for 3 replicas of tablet 5923b17ed0474cd0a4f7763ce9ea5734
I20240502 07:59:11.479380 22957 tablet_service.cc:1514] Processing DeleteTablet for tablet 5923b17ed0474cd0a4f7763ce9ea5734 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:11 UTC) from {username='slave'} at 127.0.0.1:38002
I20240502 07:59:11.479398 22566 tablet_service.cc:1514] Processing DeleteTablet for tablet 5923b17ed0474cd0a4f7763ce9ea5734 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:11 UTC) from {username='slave'} at 127.0.0.1:35572
I20240502 07:59:11.479413 22826 tablet_service.cc:1514] Processing DeleteTablet for tablet 5923b17ed0474cd0a4f7763ce9ea5734 with delete_type TABLET_DATA_DELETED (Table deleted at 2024-05-02 07:59:11 UTC) from {username='slave'} at 127.0.0.1:44098
I20240502 07:59:11.479748 1272 tablet_replica.cc:330] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85: stopping tablet replica
I20240502 07:59:11.479761 1274 tablet_replica.cc:330] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88: stopping tablet replica
I20240502 07:59:11.479892 1272 raft_consensus.cc:2229] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Raft consensus shutting down.
I20240502 07:59:11.479892 1274 raft_consensus.cc:2229] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:11.479990 1274 raft_consensus.cc:2258] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:11.480100 1272 raft_consensus.cc:2258] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:11.480321 1273 tablet_replica.cc:330] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f: stopping tablet replica
I20240502 07:59:11.480468 1273 raft_consensus.cc:2229] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 07:59:11.480566 1273 raft_consensus.cc:2258] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 07:59:11.480547 1274 ts_tablet_manager.cc:1875] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:11.480993 1272 ts_tablet_manager.cc:1875] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:11.481153 1273 ts_tablet_manager.cc:1875] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20240502 07:59:11.483209 1274 ts_tablet_manager.cc:1888] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:11.483217 1272 ts_tablet_manager.cc:1888] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:11.483284 1272 log.cc:1196] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-0/wal/wals/5923b17ed0474cd0a4f7763ce9ea5734
I20240502 07:59:11.483281 1274 log.cc:1196] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-2/wal/wals/5923b17ed0474cd0a4f7763ce9ea5734
I20240502 07:59:11.483448 1273 ts_tablet_manager.cc:1888] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.4
I20240502 07:59:11.483536 1273 log.cc:1196] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting WAL directory at /tmp/dist-test-task0LLEln/test-tmp/rebalancer_tool-test.1.DDLDuringRebalancingTest.TablesCreatedAndDeletedDuringRebalancing_1.1714636686012806-16726-0/raft_consensus-itest-cluster/ts-3/wal/wals/5923b17ed0474cd0a4f7763ce9ea5734
I20240502 07:59:11.483671 1272 ts_tablet_manager.cc:1909] T 5923b17ed0474cd0a4f7763ce9ea5734 P 8bea0fe8d8044a2685f92f7a10c9cf85: Deleting consensus metadata
I20240502 07:59:11.483798 1274 ts_tablet_manager.cc:1909] T 5923b17ed0474cd0a4f7763ce9ea5734 P 7db4178eed174225bfb24f5f9445ce88: Deleting consensus metadata
I20240502 07:59:11.483831 1273 ts_tablet_manager.cc:1909] T 5923b17ed0474cd0a4f7763ce9ea5734 P 02a92e1c084b4308b9e855a4cc2c295f: Deleting consensus metadata
I20240502 07:59:11.484339 22431 catalog_manager.cc:4830] TS 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): tablet 5923b17ed0474cd0a4f7763ce9ea5734 (table rebalancer_dynamic_table_new_name [id=30c5e8c428744d3cad4c1f5cd04104bb]) successfully deleted
I20240502 07:59:11.484474 22429 catalog_manager.cc:4830] TS 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): tablet 5923b17ed0474cd0a4f7763ce9ea5734 (table rebalancer_dynamic_table_new_name [id=30c5e8c428744d3cad4c1f5cd04104bb]) successfully deleted
I20240502 07:59:11.484627 22431 catalog_manager.cc:4830] TS 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): tablet 5923b17ed0474cd0a4f7763ce9ea5734 (table rebalancer_dynamic_table_new_name [id=30c5e8c428744d3cad4c1f5cd04104bb]) successfully deleted
W20240502 07:59:11.498803 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 07:59:11.524791 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 07:59:11.545264 1107 consensus_queue.cc:1035] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:11.548341 1107 consensus_queue.cc:1035] T 5b6b4c02a8a749d2afe291578af32a4b P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:11.570094 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20240502 07:59:11.580596 22440 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_dynamic_table"
schema {
columns {
name: "key"
type: INT64
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "a"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
}
}
I20240502 07:59:11.586257 24852 tablet_service.cc:1467] Processing CreateTablet for tablet d849da50d8cf4b21820887b673b4326b (DEFAULT_TABLE table=rebalancer_dynamic_table [id=18b7404cfd7444328cf84ffd0252747e]), partition=
I20240502 07:59:11.586529 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d849da50d8cf4b21820887b673b4326b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.588516 25115 tablet_service.cc:1467] Processing CreateTablet for tablet d849da50d8cf4b21820887b673b4326b (DEFAULT_TABLE table=rebalancer_dynamic_table [id=18b7404cfd7444328cf84ffd0252747e]), partition=
I20240502 07:59:11.588742 25115 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d849da50d8cf4b21820887b673b4326b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.590714 1189 tablet_bootstrap.cc:492] T d849da50d8cf4b21820887b673b4326b P 4654231dfe994c1a9d2e37d9c3d75699: Bootstrap starting.
I20240502 07:59:11.590811 1191 tablet_bootstrap.cc:492] T d849da50d8cf4b21820887b673b4326b P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:11.591523 1189 tablet_bootstrap.cc:654] T d849da50d8cf4b21820887b673b4326b P 4654231dfe994c1a9d2e37d9c3d75699: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.591548 1191 tablet_bootstrap.cc:654] T d849da50d8cf4b21820887b673b4326b P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.592409 1191 tablet_bootstrap.cc:492] T d849da50d8cf4b21820887b673b4326b P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:11.592480 1191 ts_tablet_manager.cc:1367] T d849da50d8cf4b21820887b673b4326b P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:11.592892 1191 raft_consensus.cc:348] T d849da50d8cf4b21820887b673b4326b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:11.592974 1189 tablet_bootstrap.cc:492] T d849da50d8cf4b21820887b673b4326b P 4654231dfe994c1a9d2e37d9c3d75699: No bootstrap required, opened a new log
I20240502 07:59:11.592991 1191 raft_consensus.cc:374] T d849da50d8cf4b21820887b673b4326b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.593019 1191 raft_consensus.cc:729] T d849da50d8cf4b21820887b673b4326b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.593040 1189 ts_tablet_manager.cc:1367] T d849da50d8cf4b21820887b673b4326b P 4654231dfe994c1a9d2e37d9c3d75699: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
I20240502 07:59:11.593114 1191 consensus_queue.cc:260] T d849da50d8cf4b21820887b673b4326b P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:11.593371 1191 ts_tablet_manager.cc:1398] T d849da50d8cf4b21820887b673b4326b P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:11.593436 1189 raft_consensus.cc:348] T d849da50d8cf4b21820887b673b4326b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:11.593524 1189 raft_consensus.cc:374] T d849da50d8cf4b21820887b673b4326b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.593555 1189 raft_consensus.cc:729] T d849da50d8cf4b21820887b673b4326b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4654231dfe994c1a9d2e37d9c3d75699, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.593653 1189 consensus_queue.cc:260] T d849da50d8cf4b21820887b673b4326b P 4654231dfe994c1a9d2e37d9c3d75699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:11.593660 25235 tablet_service.cc:1467] Processing CreateTablet for tablet d849da50d8cf4b21820887b673b4326b (DEFAULT_TABLE table=rebalancer_dynamic_table [id=18b7404cfd7444328cf84ffd0252747e]), partition=
I20240502 07:59:11.593890 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d849da50d8cf4b21820887b673b4326b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.593895 1189 ts_tablet_manager.cc:1398] T d849da50d8cf4b21820887b673b4326b P 4654231dfe994c1a9d2e37d9c3d75699: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:11.596009 1281 tablet_bootstrap.cc:492] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:11.598060 1281 tablet_bootstrap.cc:654] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.599195 1281 tablet_bootstrap.cc:492] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:11.599315 1281 ts_tablet_manager.cc:1367] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:59:11.599740 1281 raft_consensus.cc:348] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:11.599844 1281 raft_consensus.cc:374] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.599875 1281 raft_consensus.cc:729] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.600009 1281 consensus_queue.cc:260] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:11.608232 1281 ts_tablet_manager.cc:1398] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.009s user 0.001s sys 0.000s
I20240502 07:59:11.677855 1152 consensus_queue.cc:1035] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:11.680866 466 consensus_queue.cc:1035] T 4a457bdbe18d4c779f93c32c7c59d3a5 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:11.806478 466 consensus_queue.cc:1035] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:11.810217 1152 consensus_queue.cc:1035] T e036a6ba0f704f86a13e81792235e7d6 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:11.819686 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:11.833376 22436 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42052:
name: "rebalancer_extra_table_78"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 07:59:11.839035 22826 tablet_service.cc:1467] Processing CreateTablet for tablet 83f1adb927e6420e87af49655f96f666 (DEFAULT_TABLE table=rebalancer_extra_table_78 [id=4c64f9117d49483cba972c38381d15c2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.839344 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 83f1adb927e6420e87af49655f96f666. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.839576 22696 tablet_service.cc:1467] Processing CreateTablet for tablet 83f1adb927e6420e87af49655f96f666 (DEFAULT_TABLE table=rebalancer_extra_table_78 [id=4c64f9117d49483cba972c38381d15c2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.839828 22696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 83f1adb927e6420e87af49655f96f666. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.841450 1318 tablet_bootstrap.cc:492] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:11.841609 25235 tablet_service.cc:1467] Processing CreateTablet for tablet 028735b72a814937a6ca94bf47041ac8 (DEFAULT_TABLE table=rebalancer_extra_table_78 [id=4c64f9117d49483cba972c38381d15c2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.841842 25235 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 028735b72a814937a6ca94bf47041ac8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.842027 22697 tablet_service.cc:1467] Processing CreateTablet for tablet f595b6d314f7475b8a6074aa9d24f488 (DEFAULT_TABLE table=rebalancer_extra_table_78 [id=4c64f9117d49483cba972c38381d15c2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.842258 22697 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f595b6d314f7475b8a6074aa9d24f488. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.843014 1318 tablet_bootstrap.cc:654] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.843817 1281 tablet_bootstrap.cc:492] T 028735b72a814937a6ca94bf47041ac8 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:11.843994 25246 tablet_service.cc:1467] Processing CreateTablet for tablet f595b6d314f7475b8a6074aa9d24f488 (DEFAULT_TABLE table=rebalancer_extra_table_78 [id=4c64f9117d49483cba972c38381d15c2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.844139 1318 tablet_bootstrap.cc:492] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:11.844201 25246 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f595b6d314f7475b8a6074aa9d24f488. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.844208 1318 ts_tablet_manager.cc:1367] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
I20240502 07:59:11.844552 1281 tablet_bootstrap.cc:654] T 028735b72a814937a6ca94bf47041ac8 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.844621 1318 raft_consensus.cc:348] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:11.839576 22695 tablet_service.cc:1467] Processing CreateTablet for tablet 028735b72a814937a6ca94bf47041ac8 (DEFAULT_TABLE table=rebalancer_extra_table_78 [id=4c64f9117d49483cba972c38381d15c2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.845481 1318 raft_consensus.cc:374] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.845523 1318 raft_consensus.cc:729] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.845629 22695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 028735b72a814937a6ca94bf47041ac8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.845635 1318 consensus_queue.cc:260] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:11.845633 22566 tablet_service.cc:1467] Processing CreateTablet for tablet 83f1adb927e6420e87af49655f96f666 (DEFAULT_TABLE table=rebalancer_extra_table_78 [id=4c64f9117d49483cba972c38381d15c2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.845880 22566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 83f1adb927e6420e87af49655f96f666. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.845968 1318 ts_tablet_manager.cc:1398] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:11.847657 24852 tablet_service.cc:1467] Processing CreateTablet for tablet 028735b72a814937a6ca94bf47041ac8 (DEFAULT_TABLE table=rebalancer_extra_table_78 [id=4c64f9117d49483cba972c38381d15c2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.847945 24852 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 028735b72a814937a6ca94bf47041ac8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.848407 1322 tablet_bootstrap.cc:492] T 83f1adb927e6420e87af49655f96f666 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 07:59:11.849268 1322 tablet_bootstrap.cc:654] T 83f1adb927e6420e87af49655f96f666 P 8bea0fe8d8044a2685f92f7a10c9cf85: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.849983 1191 tablet_bootstrap.cc:492] T 028735b72a814937a6ca94bf47041ac8 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 07:59:11.849990 1281 tablet_bootstrap.cc:492] T 028735b72a814937a6ca94bf47041ac8 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:11.850059 1281 ts_tablet_manager.cc:1367] T 028735b72a814937a6ca94bf47041ac8 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.000s
I20240502 07:59:11.850437 1281 raft_consensus.cc:348] T 028735b72a814937a6ca94bf47041ac8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.850549 1281 raft_consensus.cc:374] T 028735b72a814937a6ca94bf47041ac8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.850580 1281 raft_consensus.cc:729] T 028735b72a814937a6ca94bf47041ac8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.850679 1281 consensus_queue.cc:260] T 028735b72a814937a6ca94bf47041ac8 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.850955 1281 ts_tablet_manager.cc:1398] T 028735b72a814937a6ca94bf47041ac8 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 07:59:11.851100 1281 tablet_bootstrap.cc:492] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69: Bootstrap starting.
I20240502 07:59:11.851246 1191 tablet_bootstrap.cc:654] T 028735b72a814937a6ca94bf47041ac8 P 6d02017318a2461796d431962fe6a537: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.852227 1191 tablet_bootstrap.cc:492] T 028735b72a814937a6ca94bf47041ac8 P 6d02017318a2461796d431962fe6a537: No bootstrap required, opened a new log
I20240502 07:59:11.852306 1191 ts_tablet_manager.cc:1367] T 028735b72a814937a6ca94bf47041ac8 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.002s user 0.002s sys 0.000s
I20240502 07:59:11.852785 1191 raft_consensus.cc:348] T 028735b72a814937a6ca94bf47041ac8 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.852883 1191 raft_consensus.cc:374] T 028735b72a814937a6ca94bf47041ac8 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.852919 1191 raft_consensus.cc:729] T 028735b72a814937a6ca94bf47041ac8 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.853032 1191 consensus_queue.cc:260] T 028735b72a814937a6ca94bf47041ac8 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.853308 1191 ts_tablet_manager.cc:1398] T 028735b72a814937a6ca94bf47041ac8 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:11.854828 22826 tablet_service.cc:1467] Processing CreateTablet for tablet f595b6d314f7475b8a6074aa9d24f488 (DEFAULT_TABLE table=rebalancer_extra_table_78 [id=4c64f9117d49483cba972c38381d15c2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 07:59:11.855063 22826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f595b6d314f7475b8a6074aa9d24f488. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 07:59:11.856791 1318 tablet_bootstrap.cc:492] T f595b6d314f7475b8a6074aa9d24f488 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 07:59:11.857491 1319 tablet_bootstrap.cc:492] T 83f1adb927e6420e87af49655f96f666 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:11.858175 1318 tablet_bootstrap.cc:654] T f595b6d314f7475b8a6074aa9d24f488 P 7db4178eed174225bfb24f5f9445ce88: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.858594 1281 tablet_bootstrap.cc:654] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.858887 1319 tablet_bootstrap.cc:654] T 83f1adb927e6420e87af49655f96f666 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.859725 1318 tablet_bootstrap.cc:492] T f595b6d314f7475b8a6074aa9d24f488 P 7db4178eed174225bfb24f5f9445ce88: No bootstrap required, opened a new log
I20240502 07:59:11.859867 1318 ts_tablet_manager.cc:1367] T f595b6d314f7475b8a6074aa9d24f488 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:59:11.860364 1318 raft_consensus.cc:348] T f595b6d314f7475b8a6074aa9d24f488 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:11.860535 1318 raft_consensus.cc:374] T f595b6d314f7475b8a6074aa9d24f488 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.860605 1318 raft_consensus.cc:729] T f595b6d314f7475b8a6074aa9d24f488 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.860775 1318 consensus_queue.cc:260] T f595b6d314f7475b8a6074aa9d24f488 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:11.861341 1318 ts_tablet_manager.cc:1398] T f595b6d314f7475b8a6074aa9d24f488 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:11.861872 1322 tablet_bootstrap.cc:492] T 83f1adb927e6420e87af49655f96f666 P 8bea0fe8d8044a2685f92f7a10c9cf85: No bootstrap required, opened a new log
I20240502 07:59:11.861948 1322 ts_tablet_manager.cc:1367] T 83f1adb927e6420e87af49655f96f666 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.014s user 0.002s sys 0.000s
I20240502 07:59:11.862336 1322 raft_consensus.cc:348] T 83f1adb927e6420e87af49655f96f666 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:11.862424 1322 raft_consensus.cc:374] T 83f1adb927e6420e87af49655f96f666 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.862459 1322 raft_consensus.cc:729] T 83f1adb927e6420e87af49655f96f666 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.862579 1322 consensus_queue.cc:260] T 83f1adb927e6420e87af49655f96f666 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:11.862803 1322 ts_tablet_manager.cc:1398] T 83f1adb927e6420e87af49655f96f666 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:11.862973 1319 tablet_bootstrap.cc:492] T 83f1adb927e6420e87af49655f96f666 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:11.863042 1319 ts_tablet_manager.cc:1367] T 83f1adb927e6420e87af49655f96f666 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20240502 07:59:11.863579 1319 raft_consensus.cc:348] T 83f1adb927e6420e87af49655f96f666 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:11.864195 1319 raft_consensus.cc:374] T 83f1adb927e6420e87af49655f96f666 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.864270 1319 raft_consensus.cc:729] T 83f1adb927e6420e87af49655f96f666 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.864403 1319 consensus_queue.cc:260] T 83f1adb927e6420e87af49655f96f666 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:11.864794 1319 ts_tablet_manager.cc:1398] T 83f1adb927e6420e87af49655f96f666 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:11.864988 1319 tablet_bootstrap.cc:492] T f595b6d314f7475b8a6074aa9d24f488 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:11.866369 1319 tablet_bootstrap.cc:654] T f595b6d314f7475b8a6074aa9d24f488 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.867463 1319 tablet_bootstrap.cc:492] T f595b6d314f7475b8a6074aa9d24f488 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:11.867535 1319 ts_tablet_manager.cc:1367] T f595b6d314f7475b8a6074aa9d24f488 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:59:11.868139 1281 tablet_bootstrap.cc:492] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69: No bootstrap required, opened a new log
I20240502 07:59:11.868218 1281 ts_tablet_manager.cc:1367] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69: Time spent bootstrapping tablet: real 0.017s user 0.001s sys 0.000s
I20240502 07:59:11.868614 1281 raft_consensus.cc:348] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:11.868719 1281 raft_consensus.cc:374] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.868752 1281 raft_consensus.cc:729] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.868842 1281 consensus_queue.cc:260] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:11.867867 1319 raft_consensus.cc:348] T f595b6d314f7475b8a6074aa9d24f488 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:11.868954 1319 raft_consensus.cc:374] T f595b6d314f7475b8a6074aa9d24f488 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.868991 1319 raft_consensus.cc:729] T f595b6d314f7475b8a6074aa9d24f488 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.869094 1281 ts_tablet_manager.cc:1398] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20240502 07:59:11.869102 1319 consensus_queue.cc:260] T f595b6d314f7475b8a6074aa9d24f488 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:11.869405 1319 ts_tablet_manager.cc:1398] T f595b6d314f7475b8a6074aa9d24f488 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 07:59:11.869565 1319 tablet_bootstrap.cc:492] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 07:59:11.870321 1319 tablet_bootstrap.cc:654] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3: Neither blocks nor log segments found. Creating new log.
I20240502 07:59:11.872241 1319 tablet_bootstrap.cc:492] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3: No bootstrap required, opened a new log
I20240502 07:59:11.872339 1319 ts_tablet_manager.cc:1367] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 07:59:11.872753 1319 raft_consensus.cc:348] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.872865 1319 raft_consensus.cc:374] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 07:59:11.872903 1319 raft_consensus.cc:729] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: FOLLOWER
I20240502 07:59:11.873008 1319 consensus_queue.cc:260] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.873212 1319 ts_tablet_manager.cc:1398] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 07:59:11.932752 32520 raft_consensus.cc:482] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:11.932861 32520 raft_consensus.cc:504] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:11.933163 32520 leader_election.cc:288] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:11.933405 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f595b6d314f7475b8a6074aa9d24f488" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88" is_pre_election: true
I20240502 07:59:11.933509 22847 raft_consensus.cc:2454] T f595b6d314f7475b8a6074aa9d24f488 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:59:11.933688 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f595b6d314f7475b8a6074aa9d24f488" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:11.933750 25202 leader_election.cc:302] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:11.933804 22717 raft_consensus.cc:2454] T f595b6d314f7475b8a6074aa9d24f488 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:59:11.933882 32520 raft_consensus.cc:2781] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:11.933933 32520 raft_consensus.cc:482] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:11.933981 32520 raft_consensus.cc:3037] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:11.934720 32520 raft_consensus.cc:504] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:11.934962 32520 leader_election.cc:288] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 07:59:11.935250 22717 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f595b6d314f7475b8a6074aa9d24f488" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:11.935372 22717 raft_consensus.cc:3037] T f595b6d314f7475b8a6074aa9d24f488 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:11.936280 22717 raft_consensus.cc:2454] T f595b6d314f7475b8a6074aa9d24f488 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:59:11.936467 22847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f595b6d314f7475b8a6074aa9d24f488" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7db4178eed174225bfb24f5f9445ce88"
I20240502 07:59:11.936496 25199 leader_election.cc:302] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:11.936568 22847 raft_consensus.cc:3037] T f595b6d314f7475b8a6074aa9d24f488 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:11.936692 32520 raft_consensus.cc:2781] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:11.936759 32520 raft_consensus.cc:686] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:59:11.936864 32520 consensus_queue.cc:237] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:11.937260 22847 raft_consensus.cc:2454] T f595b6d314f7475b8a6074aa9d24f488 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:59:11.938774 22440 catalog_manager.cc:5484] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } }
I20240502 07:59:11.980602 1325 raft_consensus.cc:482] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:11.980706 1325 raft_consensus.cc:504] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.981026 1325 leader_election.cc:288] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:59:11.981240 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "028735b72a814937a6ca94bf47041ac8" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69" is_pre_election: true
I20240502 07:59:11.981307 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "028735b72a814937a6ca94bf47041ac8" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:11.981352 25265 raft_consensus.cc:2454] T 028735b72a814937a6ca94bf47041ac8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:59:11.981417 24870 raft_consensus.cc:2454] T 028735b72a814937a6ca94bf47041ac8 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 0.
I20240502 07:59:11.981563 22652 leader_election.cc:302] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 725969d490a04fdfb3ba890c3f5096e3, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:11.981781 1325 raft_consensus.cc:2781] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:11.981854 1325 raft_consensus.cc:482] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:11.981897 1325 raft_consensus.cc:3037] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:11.982748 1325 raft_consensus.cc:504] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.982975 1325 leader_election.cc:288] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Requested vote from peers 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769), 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 07:59:11.983181 25265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "028735b72a814937a6ca94bf47041ac8" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7eced220cf1642f3873fc385b6191e69"
I20240502 07:59:11.983217 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "028735b72a814937a6ca94bf47041ac8" candidate_uuid: "725969d490a04fdfb3ba890c3f5096e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:11.983268 25265 raft_consensus.cc:3037] T 028735b72a814937a6ca94bf47041ac8 P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:11.983296 24870 raft_consensus.cc:3037] T 028735b72a814937a6ca94bf47041ac8 P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:11.983916 25265 raft_consensus.cc:2454] T 028735b72a814937a6ca94bf47041ac8 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:59:11.983920 24870 raft_consensus.cc:2454] T 028735b72a814937a6ca94bf47041ac8 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 725969d490a04fdfb3ba890c3f5096e3 in term 1.
I20240502 07:59:11.984131 22650 leader_election.cc:302] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 725969d490a04fdfb3ba890c3f5096e3; no voters:
I20240502 07:59:11.984261 1325 raft_consensus.cc:2781] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:11.984333 1325 raft_consensus.cc:686] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Becoming Leader. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Running, Role: LEADER
I20240502 07:59:11.984449 1325 consensus_queue.cc:237] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } }
I20240502 07:59:11.986569 22440 catalog_manager.cc:5484] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130). New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } }
I20240502 07:59:11.999333 32520 raft_consensus.cc:482] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:11.999419 32520 raft_consensus.cc:504] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:11.999754 32520 leader_election.cc:288] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:12.000067 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d849da50d8cf4b21820887b673b4326b" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537" is_pre_election: true
I20240502 07:59:12.000177 24870 raft_consensus.cc:2454] T d849da50d8cf4b21820887b673b4326b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:59:12.000308 25135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d849da50d8cf4b21820887b673b4326b" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699" is_pre_election: true
I20240502 07:59:12.000391 25199 leader_election.cc:302] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d02017318a2461796d431962fe6a537, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:12.000432 25135 raft_consensus.cc:2454] T d849da50d8cf4b21820887b673b4326b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 0.
I20240502 07:59:12.000546 32520 raft_consensus.cc:2781] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:12.000599 32520 raft_consensus.cc:482] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:12.000635 32520 raft_consensus.cc:3037] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:12.001433 32520 raft_consensus.cc:504] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:12.001663 32520 leader_election.cc:288] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Requested vote from peers 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459), 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 07:59:12.001881 24870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d849da50d8cf4b21820887b673b4326b" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6d02017318a2461796d431962fe6a537"
I20240502 07:59:12.001977 24870 raft_consensus.cc:3037] T d849da50d8cf4b21820887b673b4326b P 6d02017318a2461796d431962fe6a537 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:12.002795 24870 raft_consensus.cc:2454] T d849da50d8cf4b21820887b673b4326b P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:59:12.002975 25134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d849da50d8cf4b21820887b673b4326b" candidate_uuid: "7eced220cf1642f3873fc385b6191e69" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4654231dfe994c1a9d2e37d9c3d75699"
I20240502 07:59:12.003067 25134 raft_consensus.cc:3037] T d849da50d8cf4b21820887b673b4326b P 4654231dfe994c1a9d2e37d9c3d75699 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:12.003931 25134 raft_consensus.cc:2454] T d849da50d8cf4b21820887b673b4326b P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7eced220cf1642f3873fc385b6191e69 in term 1.
I20240502 07:59:12.004146 25202 leader_election.cc:302] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4654231dfe994c1a9d2e37d9c3d75699, 7eced220cf1642f3873fc385b6191e69; no voters:
I20240502 07:59:12.004305 32520 raft_consensus.cc:2781] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:12.004374 32520 raft_consensus.cc:686] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Becoming Leader. State: Replica: 7eced220cf1642f3873fc385b6191e69, State: Running, Role: LEADER
I20240502 07:59:12.004501 32520 consensus_queue.cc:237] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } }
I20240502 07:59:12.006501 22440 catalog_manager.cc:5484] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7eced220cf1642f3873fc385b6191e69 (127.16.85.136). New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } health_report { overall_health: HEALTHY } } }
W20240502 07:59:12.082929 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:12.091097 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
I20240502 07:59:12.137826 466 raft_consensus.cc:482] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 07:59:12.137954 466 raft_consensus.cc:504] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:12.138345 466 leader_election.cc:288] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:59:12.138608 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "83f1adb927e6420e87af49655f96f666" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" is_pre_election: true
I20240502 07:59:12.138729 22584 raft_consensus.cc:2454] T 83f1adb927e6420e87af49655f96f666 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:12.138760 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "83f1adb927e6420e87af49655f96f666" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3" is_pre_election: true
I20240502 07:59:12.138932 22715 raft_consensus.cc:2454] T 83f1adb927e6420e87af49655f96f666 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 0.
I20240502 07:59:12.139008 22784 leader_election.cc:302] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:12.139164 466 raft_consensus.cc:2781] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 07:59:12.139223 466 raft_consensus.cc:482] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 07:59:12.139268 466 raft_consensus.cc:3037] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:12.140031 466 raft_consensus.cc:504] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:12.140386 466 leader_election.cc:288] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Requested vote from peers 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387), 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 07:59:12.140631 22715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "83f1adb927e6420e87af49655f96f666" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "725969d490a04fdfb3ba890c3f5096e3"
I20240502 07:59:12.140635 22584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "83f1adb927e6420e87af49655f96f666" candidate_uuid: "7db4178eed174225bfb24f5f9445ce88" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85"
I20240502 07:59:12.140749 22715 raft_consensus.cc:3037] T 83f1adb927e6420e87af49655f96f666 P 725969d490a04fdfb3ba890c3f5096e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:12.140774 22584 raft_consensus.cc:3037] T 83f1adb927e6420e87af49655f96f666 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 0 FOLLOWER]: Advancing to term 1
I20240502 07:59:12.141669 22584 raft_consensus.cc:2454] T 83f1adb927e6420e87af49655f96f666 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:12.141671 22715 raft_consensus.cc:2454] T 83f1adb927e6420e87af49655f96f666 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7db4178eed174225bfb24f5f9445ce88 in term 1.
I20240502 07:59:12.141928 22784 leader_election.cc:302] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7db4178eed174225bfb24f5f9445ce88, 8bea0fe8d8044a2685f92f7a10c9cf85; no voters:
I20240502 07:59:12.142145 466 raft_consensus.cc:2781] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 07:59:12.142233 466 raft_consensus.cc:686] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Becoming Leader. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Running, Role: LEADER
I20240502 07:59:12.142357 466 consensus_queue.cc:237] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } }
I20240502 07:59:12.144809 22440 catalog_manager.cc:5484] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131). New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: UNKNOWN } } }
W20240502 07:59:12.275000 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 07:59:12.277139 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:12.288537 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:12.291026 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 07:59:12.321849 602 consensus_queue.cc:1035] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:12.324757 602 consensus_queue.cc:1035] T f595b6d314f7475b8a6074aa9d24f488 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:12.420396 1402 consensus_queue.cc:1035] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:12.428067 1402 consensus_queue.cc:1035] T d849da50d8cf4b21820887b673b4326b P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:12.465463 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:12.466945 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
I20240502 07:59:12.508538 1240 consensus_queue.cc:1035] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:12.510913 1442 consensus_queue.cc:1035] T 028735b72a814937a6ca94bf47041ac8 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:12.581276 1328 consensus_queue.cc:1035] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 07:59:12.584456 1152 consensus_queue.cc:1035] T 83f1adb927e6420e87af49655f96f666 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 07:59:12.649499 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:12.665580 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:12.696835 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:12.703989 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:12.785316 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:12.827430 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 07:59:12.830039 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:12.839380 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:12.904750 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:12.970149 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:12.994143 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20240502 07:59:13.359535 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:13.506369 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:13.548902 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:13.561463 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:13.625469 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:13.627624 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:13.721796 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:13.863309 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:13.881139 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:13.906286 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:13.912257 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:14.009991 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:14.090111 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:14.455715 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:14.577586 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:14.591490 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:14.680833 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:14.814321 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:15.009935 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:15.010401 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:15.017797 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:15.022001 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:15.110940 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:15.115618 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 77 similar messages]
W20240502 07:59:15.122689 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:15.137099 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:15.192059 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:15.238847 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:15.332290 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:15.446105 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 07:59:15.497578 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 07:59:15.502166 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 59 similar messages]
W20240502 07:59:15.502741 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:15.614313 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:15.636839 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:15.638281 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 07:59:15.638753 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:15.645814 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20240502 07:59:15.920637 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 07:59:16.029565 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:16.066864 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:16.085253 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:16.115914 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:16.155367 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:16.157462 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20240502 07:59:16.163062 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:16.212225 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:16.278347 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:16.443176 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:16.490728 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:16.546051 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:16.690001 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:16.757771 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:17.128046 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:17.166628 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:17.166705 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:17.266244 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:17.338992 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:17.370579 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:17.427192 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:17.505239 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:17.637465 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:17.712513 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:17.769423 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:17.785245 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:17.822721 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:17.832845 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:17.839948 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:17.852679 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:18.146368 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:18.146378 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:18.220049 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:18.291695 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20240502 07:59:18.410398 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:18.492352 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:18.682485 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:18.732430 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:18.740306 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:18.757992 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:18.862624 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:18.912709 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:18.953792 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:18.964265 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:19.061062 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:19.177966 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:19.199245 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:19.538551 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:19.610006 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:19.705997 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:19.817325 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:19.956426 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:20.007256 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:20.092025 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:20.105577 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:20.201898 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:20.201917 22430 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 11 similar messages]
W20240502 07:59:20.202513 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): CreateTablet RPC failed for tablet 7f8b8ce864e24201ba3d318fdce76d66: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:20.202571 22430 catalog_manager.cc:4605] Request timed out: CreateTablet RPC for tablet 7f8b8ce864e24201ba3d318fdce76d66 on TS 74f9aa91c2c448f980b512fe2e8d672e
W20240502 07:59:20.202666 22430 catalog_manager.cc:4605] Request timed out: CreateTablet RPC for tablet 4f3fb3e3b89540a6bd7c6ff623ff4ff7 on TS 74f9aa91c2c448f980b512fe2e8d672e
W20240502 07:59:20.203666 22430 catalog_manager.cc:4605] Request timed out: CreateTablet RPC for tablet 266266d3892c47d6826085bdc7fc042e on TS 74f9aa91c2c448f980b512fe2e8d672e
W20240502 07:59:20.224344 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:20.245929 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:20.252107 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 82 similar messages]
W20240502 07:59:20.252621 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:20.279623 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:20.308705 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:20.456708 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:20.470201 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:20.473644 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20240502 07:59:20.533974 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 07:59:20.558984 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 07:59:20.611702 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:20.690775 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 07:59:20.889102 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:20.894043 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:20.919764 22430 catalog_manager.cc:4605] Request timed out: CreateTablet RPC for tablet fc05622956224d2ba1d829aaf1ddea3e on TS 74f9aa91c2c448f980b512fe2e8d672e
W20240502 07:59:20.919862 22430 catalog_manager.cc:4605] Request timed out: CreateTablet RPC for tablet 68cfb914f8c140728ba025b37d7da7f5 on TS 74f9aa91c2c448f980b512fe2e8d672e
W20240502 07:59:20.919899 22430 catalog_manager.cc:4605] Request timed out: CreateTablet RPC for tablet 7ecf0142784a4dacbb61f15215dbe24c on TS 74f9aa91c2c448f980b512fe2e8d672e
W20240502 07:59:20.948921 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 07:59:20.949522 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:20.957038 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:20.965474 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:21.037484 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:21.071535 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:21.157119 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 07:59:21.251250 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 07:59:21.251749 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:21.296260 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:21.390352 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:21.415616 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:21.427968 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:21.575268 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:21.617321 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:21.741845 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:21.850368 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:21.864485 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:22.139902 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:22.318114 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:22.339643 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): CreateTablet RPC failed for tablet f1862e3480644235adaf4a58f2c45bd9: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:22.339716 22430 catalog_manager.cc:4605] Request timed out: CreateTablet RPC for tablet f1862e3480644235adaf4a58f2c45bd9 on TS 74f9aa91c2c448f980b512fe2e8d672e
W20240502 07:59:22.339787 22430 catalog_manager.cc:4605] Request timed out: CreateTablet RPC for tablet 5b7fbabf24b240879fe8f15bcfa4b377 on TS 74f9aa91c2c448f980b512fe2e8d672e
W20240502 07:59:22.454504 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:22.608950 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:22.609097 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:22.652725 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:22.720170 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:22.724748 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:22.756975 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:22.824401 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:22.941295 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:22.954924 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:23.019009 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20240502 07:59:23.074872 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:23.084534 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:23.222662 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:23.272477 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:23.430917 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:23.439752 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:23.489954 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:23.524538 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:23.534112 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:23.543632 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:23.698769 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:23.832157 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:23.928463 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:24.007597 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:24.011946 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:24.017534 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:24.039153 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:24.173045 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:24.471194 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:24.500738 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:24.787499 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:24.923826 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_DELETED RPC failed for tablet 41284d865ebb4cd18f2ce08f66343262: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:24.962818 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:24.975512 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:25.045384 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:25.050500 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:25.198974 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:25.230247 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:25.270814 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 07:59:25.314966 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:25.325747 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:25.347687 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:25.411433 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:25.412285 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:25.504591 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:25.559365 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 07:59:25.569773 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20240502 07:59:25.600023 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:25.683208 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 59 similar messages]
W20240502 07:59:25.683737 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:25.748500 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 07:59:25.749174 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:25.774663 22430 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 07:59:25.906975 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:26.013797 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:26.044204 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:26.049779 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:26.058849 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 07:59:26.059458 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:26.132542 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:26.215152 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 07:59:26.266532 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:26.275087 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:26.305830 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 07:59:26.306481 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:26.323865 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:26.339498 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:26.490711 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:26.920562 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:26.934530 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:26.999665 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:27.037613 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:27.076102 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:27.393935 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:27.447160 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:27.641059 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:27.681399 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:27.693312 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:27.885008 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:27.900413 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20240502 07:59:27.905480 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:27.909626 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:27.919607 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:27.956600 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:28.011595 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:28.185743 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:28.238675 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:28.352931 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:28.402084 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:28.449548 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:28.491436 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:28.492998 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:28.618908 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:28.655359 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:28.664407 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:28.813254 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:28.821264 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:28.905366 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:28.918403 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:29.048877 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:29.073211 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:29.241364 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:29.301959 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:29.422732 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:29.701915 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:29.721921 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:29.727653 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:30.117276 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:30.164633 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:30.178493 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:30.193710 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:30.213620 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:30.233399 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:30.345535 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:30.356951 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 07:59:30.357543 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20240502 07:59:30.373915 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:30.515766 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:30.536387 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:30.681741 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 38 similar messages]
W20240502 07:59:30.722070 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:30.726521 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 07:59:30.727180 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:30.766288 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 07:59:30.783011 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:30.864172 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:31.083792 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 07:59:31.084362 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:31.179963 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:31.208355 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:31.256716 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:31.260871 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:31.287077 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:31.313657 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 07:59:31.314218 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:31.327582 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:31.366677 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 07:59:31.374364 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:31.401438 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:31.433866 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:31.433985 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:31.542693 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:31.767191 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:31.780180 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:32.120497 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:32.192431 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:32.386582 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:32.458026 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:32.587924 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:32.667342 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:32.694690 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:32.711727 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20240502 07:59:32.720767 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:32.780117 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:32.787885 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:32.965790 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:32.968350 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:32.969273 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:33.136258 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:33.147444 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:33.218989 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:33.272465 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:33.676990 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:33.696434 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:33.738131 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:33.767916 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:33.812276 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:33.815375 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:33.825196 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:33.898784 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:33.926435 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:33.948347 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:34.002813 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:34.030400 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:34.093474 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:34.379078 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:34.474232 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:34.608453 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:34.746685 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:34.799501 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:34.814294 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:35.022704 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:35.170226 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:35.220685 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:35.281353 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:35.370187 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:35.383790 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 07:59:35.384383 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:35.395097 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:35.453531 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20240502 07:59:35.564563 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:35.619045 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:35.655303 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:35.663924 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:35.766346 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 07:59:35.802134 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 57 similar messages]
W20240502 07:59:35.839489 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 07:59:35.842211 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:35.971432 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:36.044284 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:36.097880 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:36.107450 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:36.242472 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 07:59:36.257869 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:36.329006 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:36.335726 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:36.358294 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:36.372074 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:36.386040 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:36.454121 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:36.508263 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20240502 07:59:36.555438 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 07:59:36.563539 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:36.660229 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:36.775619 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:36.793350 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:36.817548 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:37.317991 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:37.335151 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:37.360215 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:37.434562 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:37.542351 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:37.607555 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:37.694221 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:37.735183 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:37.921752 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:37.970260 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:37.989468 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:38.126132 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20240502 07:59:38.198872 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:38.229001 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:38.272153 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:38.289637 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:38.309777 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:38.485849 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:38.577850 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:38.604612 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:38.699388 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:38.727102 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:38.805684 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:38.823143 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:38.842239 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:38.864466 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:38.971108 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:38.982110 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:39.007344 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:39.084059 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:39.279309 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:39.301265 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:39.372041 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:39.726275 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:39.809832 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:39.853464 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:39.949522 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:39.981957 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:40.048197 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:40.149375 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:40.221344 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:40.309468 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:40.406659 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:40.449433 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 07:59:40.449915 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:40.491212 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:40.643157 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:40.722826 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20240502 07:59:40.777755 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:40.787765 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 37 similar messages]
W20240502 07:59:40.826287 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:40.828645 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 58 similar messages]
W20240502 07:59:40.841187 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:40.873226 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 07:59:40.909896 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:40.992082 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:41.057086 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:41.080716 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:41.109858 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:41.135812 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:41.147976 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:41.334982 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:41.378619 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 07:59:41.406975 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:41.484952 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:41.517041 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20240502 07:59:41.517684 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:41.517717 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:41.533080 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:41.627382 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:41.754225 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 07:59:41.767846 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:41.991086 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:42.174806 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:42.271944 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:42.320243 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:42.389604 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:42.430634 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:42.614378 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:42.729555 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:42.734826 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:42.765414 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:42.838781 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:42.954546 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:42.989423 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:43.144210 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:43.163928 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:43.239086 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:43.284112 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:43.298323 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:43.347352 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20240502 07:59:43.368854 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:43.510217 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:43.558221 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:43.682251 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:43.738770 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:43.759836 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:43.794991 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:43.891966 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:43.957732 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:44.027657 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:44.071699 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:44.293784 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:44.300318 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:44.325420 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:44.534127 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:44.629510 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:44.783865 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:44.797561 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:44.830874 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:44.868106 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:44.930974 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:45.086599 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:45.277493 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:45.289667 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:45.310508 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:45.321825 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:45.434711 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:45.539476 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 82 similar messages]
W20240502 07:59:45.610018 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:45.684283 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:45.705664 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:45.708860 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:45.766011 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:45.789949 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 43 similar messages]
W20240502 07:59:45.826197 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:45.830103 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20240502 07:59:45.907310 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:45.916402 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:45.919708 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 07:59:45.972131 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 42 similar messages]
W20240502 07:59:46.045094 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:46.142522 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:46.216370 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:46.316859 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:46.377153 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:46.399899 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:46.558665 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:46.688536 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 07:59:46.689087 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:46.709367 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 32 similar messages]
W20240502 07:59:46.754621 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:46.772701 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:46.914026 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:46.959515 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:46.988432 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:47.016412 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 07:59:47.110139 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:47.184324 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:47.190331 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:47.451653 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:47.567154 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:47.745571 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:47.925136 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:47.940966 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:47.953137 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:48.020594 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:48.074081 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:48.113692 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:48.145644 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:48.257273 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:48.264415 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:48.266742 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:48.390343 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:48.403685 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:48.413599 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20240502 07:59:48.625272 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:48.635859 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:48.662062 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:48.664466 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:48.789323 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:48.836230 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:49.022471 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:49.196980 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:49.284988 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:49.313438 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:49.397382 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:49.435648 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:49.478613 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:49.532253 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:49.605839 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:49.686523 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:49.795995 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:49.904269 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:49.942203 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:50.065943 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:50.302323 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:50.352753 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:50.420114 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:50.482285 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:50.580078 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:50.607360 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 77 similar messages]
W20240502 07:59:50.723729 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:50.758499 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:50.833258 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20240502 07:59:50.840638 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:50.852630 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:50.968477 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 57 similar messages]
W20240502 07:59:50.969161 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:50.996234 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 07:59:51.004318 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 07:59:51.025843 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:51.056437 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:51.100070 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:51.162202 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:51.181840 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:51.185154 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:51.200397 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:51.389710 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:51.426373 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:51.529237 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:51.786880 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 07:59:51.827133 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:51.870438 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 07:59:51.871040 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:51.894511 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:51.957326 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:52.050920 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:52.119854 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:52.122785 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:52.123559 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:52.348861 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 07:59:52.349581 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:52.355224 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:52.517773 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:52.541862 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:52.628291 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:52.752530 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:53.012583 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:53.015760 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:53.063086 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:53.093659 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:53.227463 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:53.283856 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:53.361354 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:53.399670 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:53.411413 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20240502 07:59:53.421113 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:53.449587 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:53.607712 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:53.640534 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:53.731328 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:53.785362 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:53.824961 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:53.852069 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:54.001893 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:54.228533 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:54.298969 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:54.451736 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:54.457056 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:54.466915 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:54.551895 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:54.706753 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:54.724640 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:54.842271 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:54.880373 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:54.909641 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:54.971124 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:54.984779 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:55.237800 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:55.453563 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:55.498365 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:55.544076 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:55.565161 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:55.607388 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:55.634800 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:55.656255 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 83 similar messages]
W20240502 07:59:55.713815 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20240502 07:59:55.737932 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:55.838023 22430 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:55.838761 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 5487e66cb4d8416094a1321cb989f018: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:55.870160 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:55.927933 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:55.973750 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 61 similar messages]
W20240502 07:59:55.974361 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:56.004582 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 07:59:56.057168 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:56.095538 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 07:59:56.096061 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:56.105755 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:56.204888 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:56.278597 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:56.377203 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:56.470762 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:56.596233 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:56.698064 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:56.754192 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:56.754873 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:56.788684 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 07:59:56.808284 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:56.863611 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:56.873780 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:56.941006 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 07:59:56.942066 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:57.207253 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:57.250722 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:57.360847 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 07:59:57.383649 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:57.421363 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:57.485371 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:57.739030 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_DELETED RPC failed for tablet 41284d865ebb4cd18f2ce08f66343262: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:57.821300 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 07:59:57.894317 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:57.968184 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:58.013970 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:58.041086 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:58.078316 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:58.103466 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:58.196334 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:58.240212 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20240502 07:59:58.284495 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:58.418895 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:58.426432 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:58.500342 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:58.534582 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:58.570019 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:58.644263 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:58.799499 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:58.908049 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:58.946990 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:58.969417 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:59.100586 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:59.116909 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:59.178460 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 07:59:59.207661 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 07:59:59.219691 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:59.226342 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:59.450610 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 07:59:59.605363 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 07:59:59.620743 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 07:59:59.628108 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 07:59:59.702214 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 07:59:59.750118 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:00.035135 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:00.190693 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:00.278369 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:00.323159 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 08:00:00.453819 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:00.570256 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:00.607828 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:00.631763 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:00.651696 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:00.733270 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 77 similar messages]
W20240502 08:00:00.733871 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 08:00:00.884637 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:00.910820 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20240502 08:00:00.983333 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:01.037881 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 61 similar messages]
W20240502 08:00:01.038468 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:01.041491 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:01.152408 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:00:01.187805 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:01.189242 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:01.248518 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:01.319175 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 42 similar messages]
W20240502 08:00:01.339462 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:01.404505 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:01.449115 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:01.506270 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:01.638887 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:01.672497 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:01.750670 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:01.786237 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:01.854382 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:01.902760 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:01.917208 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:01.925654 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:01.935180 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:02.034714 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:02.120921 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:02.154917 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:00:02.166590 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:00:02.307309 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:02.327035 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:02.573163 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:02.814445 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:02.846853 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:02.910020 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:00:02.911674 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:03.145164 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:03.161888 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:03.243794 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:03.244127 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:03.244835 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:03.327387 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:03.423964 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:03.511230 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20240502 08:00:03.613433 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:03.632954 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:03.755824 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:03.763258 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:03.788043 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:03.941632 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:03.957219 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:04.023185 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:04.054075 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:04.154405 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:04.173763 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:04.208489 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:04.220449 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:04.223557 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:04.282330 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:04.440487 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:04.464694 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:04.586856 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:04.781649 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:04.813416 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:04.830714 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:04.941475 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:05.226305 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:05.417649 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:05.428602 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:05.457131 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:05.545962 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:05.732939 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:05.773737 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 08:00:05.774312 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:05.834973 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20240502 08:00:05.848531 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:05.877698 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:05.921626 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:06.082572 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:06.097227 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 08:00:06.279469 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:00:06.294260 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:06.343564 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:00:06.386085 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:06.415882 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:06.424424 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:06.463861 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:06.542030 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:06.546007 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:06.547099 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:06.551033 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:06.553777 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:06.750568 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:06.766590 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:06.955886 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:06.965234 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:07.061795 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:07.117405 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:07.228638 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:07.253055 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:07.259984 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 32 similar messages]
W20240502 08:00:07.261180 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:07.346397 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:07.348500 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 32 similar messages]
W20240502 08:00:07.498334 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:07.748203 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:07.890780 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:07.917593 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:08.024714 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:08.037842 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:08.062376 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:00:08.238674 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20240502 08:00:08.300941 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:08.306557 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:08.307214 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:08.357959 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:08.471122 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:08.711967 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:08.766580 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:08.814642 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:08.865304 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:08.939122 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:08.983219 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:09.005113 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:09.009166 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:09.051033 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:09.106969 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:09.176339 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:09.260389 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:09.279911 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:09.360113 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:09.454664 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:09.467366 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:09.557690 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:09.654757 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:09.671061 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:09.725867 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:09.829396 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:10.127669 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:10.210037 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:10.315621 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:10.463852 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:10.565201 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:10.615357 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:10.629915 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:10.742985 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:10.754912 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:10.794147 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 78 similar messages]
W20240502 08:00:10.794723 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20240502 08:00:10.804860 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:10.856328 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:11.187022 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 08:00:11.249058 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:11.340819 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:11.384294 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 38 similar messages]
W20240502 08:00:11.387346 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:11.392712 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:11.436820 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:11.459275 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:11.473767 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:00:11.474294 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:11.558782 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:11.639760 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:11.640245 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:11.831758 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:11.873428 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:11.880291 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:11.912050 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:11.924757 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:11.947113 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:11.956769 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:11.974925 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:12.003688 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:12.143343 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:12.267655 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:00:12.420471 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:12.480281 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:00:12.669827 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:12.715272 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:12.791432 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:12.889012 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:12.891494 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:13.002444 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:13.144925 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:13.179304 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:13.234189 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:13.274910 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20240502 08:00:13.279101 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:13.369268 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:00:13.521003 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:13.564285 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:13.585698 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:13.778738 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:13.790609 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:13.823248 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:13.862242 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:13.869629 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:13.918568 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:14.077256 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:14.139318 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:14.414894 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:14.431066 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:14.449712 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:14.552985 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:14.649030 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:14.678714 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:14.709005 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:14.795634 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:14.796339 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:14.846959 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:15.153916 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:15.174780 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:15.363058 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:15.365985 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:15.414206 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:15.434268 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:15.521313 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20240502 08:00:15.542295 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:15.609498 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:15.702127 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:15.776332 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:15.813452 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 08:00:15.898065 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:15.909667 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:15.947480 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:16.058454 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:16.216704 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:16.238333 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 08:00:16.238934 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:16.496788 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:00:16.497434 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:16.503324 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:16.525064 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:16.565486 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:16.565806 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:00:16.569180 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:16.727257 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:16.766928 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:16.931002 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:16.964826 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:17.078624 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:17.129791 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:17.144635 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:17.164587 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:17.177369 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:17.187445 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:17.288708 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:17.511530 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:00:17.519842 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:00:17.620922 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:17.630700 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:17.805269 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:17.820102 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:17.867599 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:18.007851 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:18.217093 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:18.230509 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:18.259727 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:18.293799 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:18.308620 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20240502 08:00:18.335538 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:18.363364 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:18.577137 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:18.643543 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:18.650964 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:18.703519 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:18.793329 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:00:18.824504 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:18.941401 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:18.948804 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:19.033641 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:19.052830 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:19.069561 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:19.179484 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:19.300895 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:19.447739 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:19.486116 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:19.510197 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:19.568241 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:19.644454 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:19.768355 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:19.866335 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:19.887955 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:19.896631 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:19.973719 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:20.125726 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:20.169404 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:20.172281 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:20.683060 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:20.685359 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:20.694172 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:20.705263 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:20.764401 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20240502 08:00:20.767120 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:20.771533 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:20.853128 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:20.885972 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:20.934371 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 82 similar messages]
W20240502 08:00:21.029327 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:21.068960 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:21.290838 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:21.314710 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 59 similar messages]
W20240502 08:00:21.315403 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:21.432607 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:21.470695 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:21.569948 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:21.585274 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:00:21.637737 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:21.801739 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 08:00:21.821751 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:21.867583 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:21.893496 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:22.040951 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:22.094930 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:22.124615 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:22.207267 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:22.257161 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:22.320973 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:22.404853 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:22.406049 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:22.459082 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:22.515342 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:00:22.544914 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:22.654871 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:22.710763 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:00:22.826719 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:22.827037 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:22.959683 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:23.195195 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:23.324461 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:23.326223 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:23.383282 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:23.397616 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:23.425311 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:23.442831 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20240502 08:00:23.489953 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:23.574247 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:23.577839 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:23.839061 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:23.841552 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:23.873282 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:23.903580 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:23.957371 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:00:24.148052 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:24.171530 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:24.356343 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:24.364784 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:24.413605 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:24.476954 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:24.637827 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:24.690872 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:24.763590 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:24.775945 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:24.834988 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:24.866070 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:24.909435 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:24.958479 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:25.109189 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:25.477334 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:25.568001 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:25.576967 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:25.587313 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:25.619475 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:25.731441 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:25.853904 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:25.892686 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20240502 08:00:25.901100 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:25.931715 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:25.974779 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:25.978926 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:25.986833 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 77 similar messages]
W20240502 08:00:26.005411 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:26.012534 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
I20240502 08:00:26.328032 22433 maintenance_manager.cc:419] P a69026f7179945c0b6a4b19bfb88d16e: Scheduling FlushMRSOp(00000000000000000000000000000000): perf score=0.033366
W20240502 08:00:26.341780 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:26.349259 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
I20240502 08:00:26.360898 22432 maintenance_manager.cc:643] P a69026f7179945c0b6a4b19bfb88d16e: FlushMRSOp(00000000000000000000000000000000) complete. Timing: real 0.032s user 0.028s sys 0.004s Metrics: {"bytes_written":117462,"cfile_init":1,"dirs.queue_time_us":175,"dirs.run_cpu_time_us":229,"dirs.run_wall_time_us":904,"drs_written":1,"lbm_read_time_us":49,"lbm_reads_lt_1ms":4,"lbm_write_time_us":875,"lbm_writes_lt_1ms":35,"peak_mem_usage":0,"rows_written":325,"thread_start_us":157,"threads_started":2,"wal-append.queue_time_us":133}
I20240502 08:00:26.361966 22433 maintenance_manager.cc:419] P a69026f7179945c0b6a4b19bfb88d16e: Scheduling UndoDeltaBlockGCOp(00000000000000000000000000000000): 40192 bytes on disk
I20240502 08:00:26.363303 22432 maintenance_manager.cc:643] P a69026f7179945c0b6a4b19bfb88d16e: UndoDeltaBlockGCOp(00000000000000000000000000000000) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":69,"lbm_reads_lt_1ms":4}
W20240502 08:00:26.447882 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 08:00:26.465752 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:26.629760 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 37 similar messages]
W20240502 08:00:26.637620 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:26.675662 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:26.860807 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:00:26.967538 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:26.968842 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:27.010972 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:27.053092 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:27.085670 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:27.115365 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:27.181001 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:27.196445 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:27.279587 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:27.313472 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:27.325243 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:27.385936 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:27.522181 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:27.672897 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:00:27.726238 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:27.914638 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20240502 08:00:28.090584 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:28.188459 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:28.202659 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:28.208235 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:28.227098 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:28.234138 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:28.351878 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20240502 08:00:28.526662 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:28.532977 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:28.588487 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:28.680299 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:28.688439 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:28.693413 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:28.713164 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:28.870609 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:28.880730 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:29.077525 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:29.098071 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:29.140012 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:00:29.250751 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:29.305824 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:29.306972 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:29.418804 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:29.425555 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:29.533354 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:29.577989 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:29.746240 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:29.751965 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:29.764714 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:29.782442 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:29.801810 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:29.931828 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:30.240267 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:30.393314 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:30.549696 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:30.651863 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:30.692843 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:30.756595 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:30.790289 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:30.819885 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:30.893808 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20240502 08:00:30.985736 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:31.026837 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 83 similar messages]
W20240502 08:00:31.085685 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:31.130046 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:31.175261 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:31.201439 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:31.259727 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:31.352564 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:31.361130 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:31.407294 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:31.446427 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:31.468145 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 08:00:31.468784 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:31.548079 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:31.698040 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:00:31.699008 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:31.778251 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:31.815738 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:31.901604 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
I20240502 08:00:31.932757 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(015ed33d8c634473997eaa60e91269a2): perf score=0.033360
I20240502 08:00:31.958113 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(015ed33d8c634473997eaa60e91269a2) complete. Timing: real 0.025s user 0.024s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"compiler_manager_pool.queue_time_us":160,"dirs.queue_time_us":154,"dirs.run_cpu_time_us":170,"dirs.run_wall_time_us":1342,"drs_written":1,"lbm_read_time_us":74,"lbm_reads_lt_1ms":4,"lbm_write_time_us":391,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":220,"threads_started":3,"wal-append.queue_time_us":171}
I20240502 08:00:31.959901 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(015ed33d8c634473997eaa60e91269a2): 4840 bytes on disk
I20240502 08:00:31.960314 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(015ed33d8c634473997eaa60e91269a2) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":40,"lbm_reads_lt_1ms":4}
I20240502 08:00:31.961869 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(313eef41dee34cf2863004373a449156): perf score=0.033367
I20240502 08:00:31.979473 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(313eef41dee34cf2863004373a449156) complete. Timing: real 0.017s user 0.015s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":34,"dirs.run_cpu_time_us":193,"dirs.run_wall_time_us":1621,"drs_written":1,"lbm_read_time_us":36,"lbm_reads_lt_1ms":4,"lbm_write_time_us":485,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":106,"threads_started":1,"wal-append.queue_time_us":167}
I20240502 08:00:31.981161 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(313eef41dee34cf2863004373a449156): 4925 bytes on disk
I20240502 08:00:31.981528 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(313eef41dee34cf2863004373a449156) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":61,"lbm_reads_lt_1ms":4}
I20240502 08:00:31.983717 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling FlushMRSOp(313eef41dee34cf2863004373a449156): perf score=0.033375
I20240502 08:00:32.027128 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling FlushMRSOp(e5d5f67e026344998d5fb079f532d076): perf score=0.033386
W20240502 08:00:32.039935 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
I20240502 08:00:32.050921 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling FlushMRSOp(015ed33d8c634473997eaa60e91269a2): perf score=0.033393
I20240502 08:00:32.050921 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: FlushMRSOp(313eef41dee34cf2863004373a449156) complete. Timing: real 0.067s user 0.027s sys 0.005s Metrics: {"bytes_written":6144,"cfile_init":1,"compiler_manager_pool.queue_time_us":3894,"dirs.queue_time_us":166,"dirs.run_cpu_time_us":162,"dirs.run_wall_time_us":1643,"drs_written":1,"lbm_read_time_us":69,"lbm_reads_lt_1ms":4,"lbm_write_time_us":594,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":220,"threads_started":3,"wal-append.queue_time_us":25157}
I20240502 08:00:32.053632 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling UndoDeltaBlockGCOp(313eef41dee34cf2863004373a449156): 4925 bytes on disk
I20240502 08:00:32.054190 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: UndoDeltaBlockGCOp(313eef41dee34cf2863004373a449156) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":68,"lbm_reads_lt_1ms":4}
I20240502 08:00:32.110535 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: FlushMRSOp(e5d5f67e026344998d5fb079f532d076) complete. Timing: real 0.083s user 0.029s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"compiler_manager_pool.queue_time_us":4525,"dirs.queue_time_us":15154,"dirs.run_cpu_time_us":178,"dirs.run_wall_time_us":815,"drs_written":1,"lbm_read_time_us":87,"lbm_reads_lt_1ms":4,"lbm_write_time_us":590,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":189,"threads_started":3,"wal-append.queue_time_us":36364}
I20240502 08:00:32.113248 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling UndoDeltaBlockGCOp(e5d5f67e026344998d5fb079f532d076): 4586 bytes on disk
I20240502 08:00:32.113884 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: UndoDeltaBlockGCOp(e5d5f67e026344998d5fb079f532d076) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":79,"lbm_reads_lt_1ms":4}
W20240502 08:00:32.114480 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:00:32.116319 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
I20240502 08:00:32.116756 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling FlushMRSOp(313eef41dee34cf2863004373a449156): perf score=0.033410
W20240502 08:00:32.131843 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
I20240502 08:00:32.152134 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: FlushMRSOp(313eef41dee34cf2863004373a449156) complete. Timing: real 0.035s user 0.018s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":5130,"dirs.run_cpu_time_us":200,"dirs.run_wall_time_us":1533,"drs_written":1,"lbm_read_time_us":62,"lbm_reads_lt_1ms":4,"lbm_write_time_us":538,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":104,"threads_started":1,"wal-append.queue_time_us":7140}
I20240502 08:00:32.154278 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling UndoDeltaBlockGCOp(313eef41dee34cf2863004373a449156): 4925 bytes on disk
I20240502 08:00:32.154666 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: UndoDeltaBlockGCOp(313eef41dee34cf2863004373a449156) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":87,"lbm_reads_lt_1ms":4}
W20240502 08:00:32.160238 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
I20240502 08:00:32.172541 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: FlushMRSOp(015ed33d8c634473997eaa60e91269a2) complete. Timing: real 0.121s user 0.033s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"compiler_manager_pool.queue_time_us":236,"dirs.queue_time_us":23627,"dirs.run_cpu_time_us":154,"dirs.run_wall_time_us":6491,"drs_written":1,"lbm_read_time_us":73,"lbm_reads_lt_1ms":4,"lbm_write_time_us":3787,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":22,"peak_mem_usage":0,"rows_written":676,"thread_start_us":379,"threads_started":3,"wal-append.queue_time_us":16626}
I20240502 08:00:32.175097 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling UndoDeltaBlockGCOp(015ed33d8c634473997eaa60e91269a2): 4840 bytes on disk
I20240502 08:00:32.175693 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: UndoDeltaBlockGCOp(015ed33d8c634473997eaa60e91269a2) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":61,"lbm_reads_lt_1ms":4}
W20240502 08:00:32.225158 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:32.447697 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:32.542093 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:32.566562 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
I20240502 08:00:32.660212 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling FlushMRSOp(e24e4e639b0649e3ac135ac9358a0398): perf score=0.033343
I20240502 08:00:32.672755 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: FlushMRSOp(e24e4e639b0649e3ac135ac9358a0398) complete. Timing: real 0.012s user 0.011s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":128,"dirs.run_cpu_time_us":173,"dirs.run_wall_time_us":1345,"drs_written":1,"lbm_read_time_us":44,"lbm_reads_lt_1ms":4,"lbm_write_time_us":485,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":126,"threads_started":2,"wal-append.queue_time_us":118}
I20240502 08:00:32.674402 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling UndoDeltaBlockGCOp(e24e4e639b0649e3ac135ac9358a0398): 4840 bytes on disk
I20240502 08:00:32.674674 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: UndoDeltaBlockGCOp(e24e4e639b0649e3ac135ac9358a0398) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":40,"lbm_reads_lt_1ms":4}
I20240502 08:00:32.679944 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling FlushMRSOp(68f96ca3354645fd91f666109eeaf847): perf score=0.033348
I20240502 08:00:32.693174 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: FlushMRSOp(68f96ca3354645fd91f666109eeaf847) complete. Timing: real 0.013s user 0.013s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":144,"dirs.run_cpu_time_us":160,"dirs.run_wall_time_us":1169,"drs_written":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4,"lbm_write_time_us":467,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":122,"threads_started":2,"wal-append.queue_time_us":117}
I20240502 08:00:32.694541 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling UndoDeltaBlockGCOp(68f96ca3354645fd91f666109eeaf847): 4925 bytes on disk
I20240502 08:00:32.694814 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: UndoDeltaBlockGCOp(68f96ca3354645fd91f666109eeaf847) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":40,"lbm_reads_lt_1ms":4}
I20240502 08:00:32.738603 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(ca85b40df34d4e1c91a55d7dd69e1494): perf score=0.033363
I20240502 08:00:32.751067 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(ca85b40df34d4e1c91a55d7dd69e1494) complete. Timing: real 0.012s user 0.011s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":159,"dirs.run_cpu_time_us":169,"dirs.run_wall_time_us":1363,"drs_written":1,"lbm_read_time_us":43,"lbm_reads_lt_1ms":4,"lbm_write_time_us":459,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":144,"threads_started":2,"wal-append.queue_time_us":116}
I20240502 08:00:32.753044 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(68f96ca3354645fd91f666109eeaf847): perf score=0.033366
I20240502 08:00:32.765885 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(68f96ca3354645fd91f666109eeaf847) complete. Timing: real 0.013s user 0.011s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":38,"dirs.run_cpu_time_us":160,"dirs.run_wall_time_us":1022,"drs_written":1,"lbm_read_time_us":55,"lbm_reads_lt_1ms":4,"lbm_write_time_us":444,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":94,"threads_started":1,"wal-append.queue_time_us":166}
I20240502 08:00:32.767786 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(68f96ca3354645fd91f666109eeaf847): 4925 bytes on disk
I20240502 08:00:32.768447 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(68f96ca3354645fd91f666109eeaf847) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":154,"lbm_reads_lt_1ms":4}
I20240502 08:00:32.769944 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(ca85b40df34d4e1c91a55d7dd69e1494): 4586 bytes on disk
I20240502 08:00:32.770221 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(ca85b40df34d4e1c91a55d7dd69e1494) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":41,"lbm_reads_lt_1ms":4}
I20240502 08:00:32.811174 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling FlushMRSOp(e24e4e639b0649e3ac135ac9358a0398): perf score=0.033385
I20240502 08:00:32.824213 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: FlushMRSOp(e24e4e639b0649e3ac135ac9358a0398) complete. Timing: real 0.013s user 0.011s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":167,"dirs.run_cpu_time_us":162,"dirs.run_wall_time_us":1291,"drs_written":1,"lbm_read_time_us":34,"lbm_reads_lt_1ms":4,"lbm_write_time_us":518,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":149,"threads_started":2,"wal-append.queue_time_us":134}
I20240502 08:00:32.825913 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling UndoDeltaBlockGCOp(e24e4e639b0649e3ac135ac9358a0398): 4840 bytes on disk
I20240502 08:00:32.826539 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: UndoDeltaBlockGCOp(e24e4e639b0649e3ac135ac9358a0398) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":50,"lbm_reads_lt_1ms":4}
W20240502 08:00:32.826949 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 08:00:32.827975 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling FlushMRSOp(68f96ca3354645fd91f666109eeaf847): perf score=0.033389
W20240502 08:00:32.836033 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
I20240502 08:00:32.840703 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: FlushMRSOp(68f96ca3354645fd91f666109eeaf847) complete. Timing: real 0.013s user 0.012s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":39,"dirs.run_cpu_time_us":183,"dirs.run_wall_time_us":1309,"drs_written":1,"lbm_read_time_us":35,"lbm_reads_lt_1ms":4,"lbm_write_time_us":453,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":97,"threads_started":1,"wal-append.queue_time_us":192}
I20240502 08:00:32.842137 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling UndoDeltaBlockGCOp(68f96ca3354645fd91f666109eeaf847): 4925 bytes on disk
I20240502 08:00:32.842460 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: UndoDeltaBlockGCOp(68f96ca3354645fd91f666109eeaf847) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":62,"lbm_reads_lt_1ms":4}
W20240502 08:00:32.903299 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:32.942940 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:00:32.955403 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:00:32.960223 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:33.142088 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:33.274863 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:33.377045 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:33.379179 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:33.437036 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20240502 08:00:33.466296 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:33.509382 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:33.529078 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:33.536463 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
I20240502 08:00:33.681661 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling FlushMRSOp(88c1176b415c412fb2c824f277e4d3c8): perf score=0.033360
I20240502 08:00:33.693260 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: FlushMRSOp(88c1176b415c412fb2c824f277e4d3c8) complete. Timing: real 0.011s user 0.006s sys 0.004s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":145,"dirs.run_cpu_time_us":150,"dirs.run_wall_time_us":1504,"drs_written":1,"lbm_read_time_us":51,"lbm_reads_lt_1ms":4,"lbm_write_time_us":416,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":124,"threads_started":2,"wal-append.queue_time_us":119}
W20240502 08:00:33.694937 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 08:00:33.695071 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling UndoDeltaBlockGCOp(88c1176b415c412fb2c824f277e4d3c8): 4925 bytes on disk
I20240502 08:00:33.695384 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: UndoDeltaBlockGCOp(88c1176b415c412fb2c824f277e4d3c8) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":46,"lbm_reads_lt_1ms":4}
I20240502 08:00:33.701658 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling FlushMRSOp(a5374313bbba4b10904e0ea18851c07d): perf score=0.033363
I20240502 08:00:33.712821 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: FlushMRSOp(a5374313bbba4b10904e0ea18851c07d) complete. Timing: real 0.011s user 0.011s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":126,"dirs.run_cpu_time_us":152,"dirs.run_wall_time_us":1147,"drs_written":1,"lbm_read_time_us":30,"lbm_reads_lt_1ms":4,"lbm_write_time_us":377,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":108,"threads_started":2,"wal-append.queue_time_us":99}
I20240502 08:00:33.714133 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling UndoDeltaBlockGCOp(a5374313bbba4b10904e0ea18851c07d): 4840 bytes on disk
I20240502 08:00:33.714591 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: UndoDeltaBlockGCOp(a5374313bbba4b10904e0ea18851c07d) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4}
W20240502 08:00:33.727228 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:33.736536 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
I20240502 08:00:33.777725 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(a5374313bbba4b10904e0ea18851c07d): perf score=0.033383
W20240502 08:00:33.806012 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
I20240502 08:00:33.812883 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(a5374313bbba4b10904e0ea18851c07d) complete. Timing: real 0.035s user 0.012s sys 0.002s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":649,"dirs.run_cpu_time_us":166,"dirs.run_wall_time_us":1457,"drs_written":1,"lbm_read_time_us":67,"lbm_reads_lt_1ms":4,"lbm_write_time_us":443,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":132,"threads_started":2,"wal-append.queue_time_us":5829}
I20240502 08:00:33.814988 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(55fb5c7475b14e0eaced5b8de1792641): perf score=0.033393
W20240502 08:00:33.824489 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
I20240502 08:00:33.828413 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(55fb5c7475b14e0eaced5b8de1792641) complete. Timing: real 0.013s user 0.012s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":38,"dirs.run_cpu_time_us":135,"dirs.run_wall_time_us":1175,"drs_written":1,"lbm_read_time_us":31,"lbm_reads_lt_1ms":4,"lbm_write_time_us":512,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":55,"threads_started":1,"wal-append.queue_time_us":122}
I20240502 08:00:33.830432 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(a5374313bbba4b10904e0ea18851c07d): 4840 bytes on disk
I20240502 08:00:33.830698 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(a5374313bbba4b10904e0ea18851c07d) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":42,"lbm_reads_lt_1ms":4}
I20240502 08:00:33.832310 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(55fb5c7475b14e0eaced5b8de1792641): 4586 bytes on disk
I20240502 08:00:33.832715 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(55fb5c7475b14e0eaced5b8de1792641) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":70,"lbm_reads_lt_1ms":4}
I20240502 08:00:33.849463 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling FlushMRSOp(88c1176b415c412fb2c824f277e4d3c8): perf score=0.033403
I20240502 08:00:33.862798 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: FlushMRSOp(88c1176b415c412fb2c824f277e4d3c8) complete. Timing: real 0.013s user 0.012s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":128,"dirs.run_cpu_time_us":130,"dirs.run_wall_time_us":1270,"drs_written":1,"lbm_read_time_us":48,"lbm_reads_lt_1ms":4,"lbm_write_time_us":543,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":114,"threads_started":2,"wal-append.queue_time_us":110}
I20240502 08:00:33.864145 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling FlushMRSOp(55fb5c7475b14e0eaced5b8de1792641): perf score=0.033406
I20240502 08:00:33.875072 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: FlushMRSOp(55fb5c7475b14e0eaced5b8de1792641) complete. Timing: real 0.011s user 0.007s sys 0.004s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":33,"dirs.run_cpu_time_us":146,"dirs.run_wall_time_us":1356,"drs_written":1,"lbm_read_time_us":39,"lbm_reads_lt_1ms":4,"lbm_write_time_us":383,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":67,"threads_started":1,"wal-append.queue_time_us":152}
I20240502 08:00:33.876426 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling UndoDeltaBlockGCOp(88c1176b415c412fb2c824f277e4d3c8): 4925 bytes on disk
I20240502 08:00:33.876778 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: UndoDeltaBlockGCOp(88c1176b415c412fb2c824f277e4d3c8) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":55,"lbm_reads_lt_1ms":4}
I20240502 08:00:33.878181 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling UndoDeltaBlockGCOp(55fb5c7475b14e0eaced5b8de1792641): 4586 bytes on disk
I20240502 08:00:33.878448 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: UndoDeltaBlockGCOp(55fb5c7475b14e0eaced5b8de1792641) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":38,"lbm_reads_lt_1ms":4}
W20240502 08:00:34.003127 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:34.047585 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:34.097878 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:34.102039 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:34.170712 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:00:34.188720 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:34.267392 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:34.451671 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:34.644537 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:34.687973 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:34.710619 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:34.781095 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:34.811681 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
I20240502 08:00:34.841264 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(6dbf36c0f044466ab13a53bfe0d6cb53): perf score=0.033364
I20240502 08:00:34.854668 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(6dbf36c0f044466ab13a53bfe0d6cb53) complete. Timing: real 0.013s user 0.012s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":144,"dirs.run_cpu_time_us":154,"dirs.run_wall_time_us":1337,"drs_written":1,"lbm_read_time_us":37,"lbm_reads_lt_1ms":4,"lbm_write_time_us":444,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":124,"threads_started":2,"wal-append.queue_time_us":111}
I20240502 08:00:34.856228 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(6dbf36c0f044466ab13a53bfe0d6cb53): 4925 bytes on disk
I20240502 08:00:34.856616 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(6dbf36c0f044466ab13a53bfe0d6cb53) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":73,"lbm_reads_lt_1ms":4}
I20240502 08:00:34.857964 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(abfc26b86b17464fa7cf0b13ab73861c): perf score=0.033363
I20240502 08:00:34.868531 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(abfc26b86b17464fa7cf0b13ab73861c) complete. Timing: real 0.010s user 0.009s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":39,"dirs.run_cpu_time_us":181,"dirs.run_wall_time_us":1214,"drs_written":1,"lbm_read_time_us":43,"lbm_reads_lt_1ms":4,"lbm_write_time_us":389,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":55,"threads_started":1,"wal-append.queue_time_us":113}
I20240502 08:00:34.869865 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(abfc26b86b17464fa7cf0b13ab73861c): 4840 bytes on disk
I20240502 08:00:34.870152 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(abfc26b86b17464fa7cf0b13ab73861c) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":40,"lbm_reads_lt_1ms":4}
I20240502 08:00:34.886601 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling FlushMRSOp(6dbf36c0f044466ab13a53bfe0d6cb53): perf score=0.033375
I20240502 08:00:34.898041 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: FlushMRSOp(6dbf36c0f044466ab13a53bfe0d6cb53) complete. Timing: real 0.011s user 0.010s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":117,"dirs.run_cpu_time_us":128,"dirs.run_wall_time_us":1261,"drs_written":1,"lbm_read_time_us":28,"lbm_reads_lt_1ms":4,"lbm_write_time_us":369,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":119,"threads_started":2,"wal-append.queue_time_us":112}
I20240502 08:00:34.899237 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling UndoDeltaBlockGCOp(6dbf36c0f044466ab13a53bfe0d6cb53): 4925 bytes on disk
I20240502 08:00:34.899503 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: UndoDeltaBlockGCOp(6dbf36c0f044466ab13a53bfe0d6cb53) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":39,"lbm_reads_lt_1ms":4}
W20240502 08:00:34.944578 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
I20240502 08:00:34.953070 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling FlushMRSOp(9ca2e5b5a0d24bcc9d1b69f998a0d928): perf score=0.033396
I20240502 08:00:34.964955 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: FlushMRSOp(9ca2e5b5a0d24bcc9d1b69f998a0d928) complete. Timing: real 0.012s user 0.011s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":134,"dirs.run_cpu_time_us":164,"dirs.run_wall_time_us":1231,"drs_written":1,"lbm_read_time_us":38,"lbm_reads_lt_1ms":4,"lbm_write_time_us":511,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":137,"threads_started":2,"wal-append.queue_time_us":130}
I20240502 08:00:34.966765 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling UndoDeltaBlockGCOp(9ca2e5b5a0d24bcc9d1b69f998a0d928): 4586 bytes on disk
I20240502 08:00:34.967106 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: UndoDeltaBlockGCOp(9ca2e5b5a0d24bcc9d1b69f998a0d928) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":65,"lbm_reads_lt_1ms":4}
I20240502 08:00:34.969605 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling FlushMRSOp(abfc26b86b17464fa7cf0b13ab73861c): perf score=0.033395
I20240502 08:00:34.973423 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling FlushMRSOp(9ca2e5b5a0d24bcc9d1b69f998a0d928): perf score=0.033396
I20240502 08:00:34.982384 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: FlushMRSOp(abfc26b86b17464fa7cf0b13ab73861c) complete. Timing: real 0.013s user 0.012s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":37,"dirs.run_cpu_time_us":156,"dirs.run_wall_time_us":1225,"drs_written":1,"lbm_read_time_us":28,"lbm_reads_lt_1ms":4,"lbm_write_time_us":477,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":70,"threads_started":1,"wal-append.queue_time_us":154}
I20240502 08:00:34.984719 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling UndoDeltaBlockGCOp(abfc26b86b17464fa7cf0b13ab73861c): 4840 bytes on disk
I20240502 08:00:34.985160 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: UndoDeltaBlockGCOp(abfc26b86b17464fa7cf0b13ab73861c) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":56,"lbm_reads_lt_1ms":4}
I20240502 08:00:34.989414 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: FlushMRSOp(9ca2e5b5a0d24bcc9d1b69f998a0d928) complete. Timing: real 0.015s user 0.013s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":755,"dirs.run_cpu_time_us":144,"dirs.run_wall_time_us":1085,"drs_written":1,"lbm_read_time_us":33,"lbm_reads_lt_1ms":4,"lbm_write_time_us":483,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":109,"threads_started":2,"wal-append.queue_time_us":105}
I20240502 08:00:34.991197 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling UndoDeltaBlockGCOp(9ca2e5b5a0d24bcc9d1b69f998a0d928): 4586 bytes on disk
I20240502 08:00:34.991873 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: UndoDeltaBlockGCOp(9ca2e5b5a0d24bcc9d1b69f998a0d928) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":361,"lbm_reads_lt_1ms":4}
W20240502 08:00:35.074542 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:35.181181 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:35.212790 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:35.247388 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:35.328925 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:35.403007 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 08:00:35.626147 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(cceaa8887081401dbc9154d265004c7b): perf score=0.033367
I20240502 08:00:35.638837 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(cceaa8887081401dbc9154d265004c7b) complete. Timing: real 0.013s user 0.012s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":132,"dirs.run_cpu_time_us":192,"dirs.run_wall_time_us":1503,"drs_written":1,"lbm_read_time_us":29,"lbm_reads_lt_1ms":4,"lbm_write_time_us":498,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":118,"threads_started":2,"wal-append.queue_time_us":108}
I20240502 08:00:35.640465 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(ee06ff655f9c46c1bb06af2dd539fde7): perf score=0.033369
I20240502 08:00:35.651881 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(ee06ff655f9c46c1bb06af2dd539fde7) complete. Timing: real 0.011s user 0.010s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":37,"dirs.run_cpu_time_us":107,"dirs.run_wall_time_us":1133,"drs_written":1,"lbm_read_time_us":33,"lbm_reads_lt_1ms":4,"lbm_write_time_us":425,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":92,"threads_started":1,"wal-append.queue_time_us":169}
I20240502 08:00:35.653225 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(cceaa8887081401dbc9154d265004c7b): 4925 bytes on disk
I20240502 08:00:35.653528 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(cceaa8887081401dbc9154d265004c7b) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":41,"lbm_reads_lt_1ms":4}
I20240502 08:00:35.654626 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(ee06ff655f9c46c1bb06af2dd539fde7): 4840 bytes on disk
I20240502 08:00:35.654968 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling FlushMRSOp(cceaa8887081401dbc9154d265004c7b): perf score=0.033375
I20240502 08:00:35.655055 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(ee06ff655f9c46c1bb06af2dd539fde7) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":61,"lbm_reads_lt_1ms":4}
I20240502 08:00:35.670207 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: FlushMRSOp(cceaa8887081401dbc9154d265004c7b) complete. Timing: real 0.015s user 0.012s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":119,"dirs.run_cpu_time_us":153,"dirs.run_wall_time_us":1314,"drs_written":1,"lbm_read_time_us":30,"lbm_reads_lt_1ms":4,"lbm_write_time_us":370,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":113,"threads_started":2,"wal-append.queue_time_us":506}
I20240502 08:00:35.672603 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling UndoDeltaBlockGCOp(cceaa8887081401dbc9154d265004c7b): 4925 bytes on disk
I20240502 08:00:35.673174 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: UndoDeltaBlockGCOp(cceaa8887081401dbc9154d265004c7b) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":62,"lbm_reads_lt_1ms":4}
W20240502 08:00:35.690413 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 08:00:35.741083 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling FlushMRSOp(cfff02438656430686be0f03f5c0b8ab): perf score=0.033396
I20240502 08:00:35.747330 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling FlushMRSOp(ee06ff655f9c46c1bb06af2dd539fde7): perf score=0.033399
I20240502 08:00:35.757045 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: FlushMRSOp(cfff02438656430686be0f03f5c0b8ab) complete. Timing: real 0.016s user 0.013s sys 0.001s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":155,"dirs.run_cpu_time_us":149,"dirs.run_wall_time_us":1466,"drs_written":1,"lbm_read_time_us":47,"lbm_reads_lt_1ms":4,"lbm_write_time_us":459,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":133,"threads_started":2,"wal-append.queue_time_us":139}
I20240502 08:00:35.758883 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling UndoDeltaBlockGCOp(cfff02438656430686be0f03f5c0b8ab): 4584 bytes on disk
I20240502 08:00:35.759277 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: UndoDeltaBlockGCOp(cfff02438656430686be0f03f5c0b8ab) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4}
I20240502 08:00:35.763268 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: FlushMRSOp(ee06ff655f9c46c1bb06af2dd539fde7) complete. Timing: real 0.016s user 0.013s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":2713,"dirs.run_cpu_time_us":179,"dirs.run_wall_time_us":1077,"drs_written":1,"lbm_read_time_us":38,"lbm_reads_lt_1ms":4,"lbm_write_time_us":638,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":123,"threads_started":2,"wal-append.queue_time_us":128}
I20240502 08:00:35.765322 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling UndoDeltaBlockGCOp(ee06ff655f9c46c1bb06af2dd539fde7): 4840 bytes on disk
I20240502 08:00:35.765731 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: UndoDeltaBlockGCOp(ee06ff655f9c46c1bb06af2dd539fde7) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4}
W20240502 08:00:35.789988 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:35.845573 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:35.851423 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:35.893911 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:35.971836 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:36.022760 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:36.058498 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 08:00:36.088153 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20240502 08:00:36.104801 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:36.121634 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:36.172050 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:36.176862 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:36.185721 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:36.274266 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:36.365391 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
I20240502 08:00:36.416814 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(9d2283e5d28347519b38e876d840608e): perf score=0.033366
I20240502 08:00:36.429998 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling FlushMRSOp(e25ba5cc77334a3595a4575fa98476f0): perf score=0.033368
I20240502 08:00:36.430750 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(9d2283e5d28347519b38e876d840608e) complete. Timing: real 0.014s user 0.012s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":124,"drs_written":1,"lbm_read_time_us":41,"lbm_reads_lt_1ms":4,"lbm_write_time_us":392,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":105,"threads_started":2,"wal-append.queue_time_us":94}
I20240502 08:00:36.434981 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(9d2283e5d28347519b38e876d840608e): 4840 bytes on disk
I20240502 08:00:36.435313 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(9d2283e5d28347519b38e876d840608e) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":64,"lbm_reads_lt_1ms":4}
I20240502 08:00:36.443477 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: FlushMRSOp(e25ba5cc77334a3595a4575fa98476f0) complete. Timing: real 0.013s user 0.012s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":131,"dirs.run_cpu_time_us":141,"dirs.run_wall_time_us":2852,"drs_written":1,"lbm_read_time_us":40,"lbm_reads_lt_1ms":4,"lbm_write_time_us":545,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":108,"threads_started":2,"wal-append.queue_time_us":106}
I20240502 08:00:36.445363 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling UndoDeltaBlockGCOp(e25ba5cc77334a3595a4575fa98476f0): 4925 bytes on disk
I20240502 08:00:36.445816 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: UndoDeltaBlockGCOp(e25ba5cc77334a3595a4575fa98476f0) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":63,"lbm_reads_lt_1ms":4}
I20240502 08:00:36.515074 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling FlushMRSOp(28d5d20938a4472caf029c8d1ac89b82): perf score=0.033391
I20240502 08:00:36.521046 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling FlushMRSOp(28d5d20938a4472caf029c8d1ac89b82): perf score=0.033396
I20240502 08:00:36.528597 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: FlushMRSOp(28d5d20938a4472caf029c8d1ac89b82) complete. Timing: real 0.013s user 0.010s sys 0.002s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":128,"dirs.run_cpu_time_us":201,"dirs.run_wall_time_us":1268,"drs_written":1,"lbm_read_time_us":29,"lbm_reads_lt_1ms":4,"lbm_write_time_us":461,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":130,"threads_started":2,"wal-append.queue_time_us":366}
I20240502 08:00:36.530946 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling UndoDeltaBlockGCOp(28d5d20938a4472caf029c8d1ac89b82): 4586 bytes on disk
I20240502 08:00:36.531325 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: UndoDeltaBlockGCOp(28d5d20938a4472caf029c8d1ac89b82) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":64,"lbm_reads_lt_1ms":4}
I20240502 08:00:36.535315 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: FlushMRSOp(28d5d20938a4472caf029c8d1ac89b82) complete. Timing: real 0.014s user 0.011s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":148,"dirs.run_cpu_time_us":167,"dirs.run_wall_time_us":1103,"drs_written":1,"lbm_read_time_us":43,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1497,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":22,"peak_mem_usage":0,"rows_written":635,"thread_start_us":128,"threads_started":2,"wal-append.queue_time_us":118}
I20240502 08:00:36.536669 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling UndoDeltaBlockGCOp(28d5d20938a4472caf029c8d1ac89b82): 4586 bytes on disk
I20240502 08:00:36.536940 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: UndoDeltaBlockGCOp(28d5d20938a4472caf029c8d1ac89b82) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":40,"lbm_reads_lt_1ms":4}
W20240502 08:00:36.571991 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:36.575644 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 59 similar messages]
W20240502 08:00:36.576191 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:36.642500 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:36.766494 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:36.791718 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:36.821795 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 42 similar messages]
W20240502 08:00:36.857107 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:36.960301 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:37.118906 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:37.165241 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:37.260870 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:00:37.297921 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:37.299608 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:37.315997 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 08:00:37.452576 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling FlushMRSOp(a98f26c1f53f403f9a9f6ae994cd3caa): perf score=0.033333
I20240502 08:00:37.464068 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: FlushMRSOp(a98f26c1f53f403f9a9f6ae994cd3caa) complete. Timing: real 0.011s user 0.010s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":134,"dirs.run_cpu_time_us":161,"dirs.run_wall_time_us":1608,"drs_written":1,"lbm_read_time_us":37,"lbm_reads_lt_1ms":4,"lbm_write_time_us":463,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":123,"threads_started":2,"wal-append.queue_time_us":117}
I20240502 08:00:37.465492 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling UndoDeltaBlockGCOp(a98f26c1f53f403f9a9f6ae994cd3caa): 4586 bytes on disk
I20240502 08:00:37.465798 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: UndoDeltaBlockGCOp(a98f26c1f53f403f9a9f6ae994cd3caa) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":39,"lbm_reads_lt_1ms":4}
I20240502 08:00:37.538659 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling FlushMRSOp(f1fa312fe8e44d23970c5e3ba98c2c58): perf score=0.033358
I20240502 08:00:37.543711 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling FlushMRSOp(c53e8b3bd774466dbafe4994472975bc): perf score=0.033357
I20240502 08:00:37.552336 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: FlushMRSOp(f1fa312fe8e44d23970c5e3ba98c2c58) complete. Timing: real 0.013s user 0.012s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":3253,"dirs.run_cpu_time_us":195,"dirs.run_wall_time_us":1328,"drs_written":1,"lbm_read_time_us":53,"lbm_reads_lt_1ms":4,"lbm_write_time_us":533,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":115,"threads_started":2,"wal-append.queue_time_us":231}
I20240502 08:00:37.554230 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling UndoDeltaBlockGCOp(f1fa312fe8e44d23970c5e3ba98c2c58): 4925 bytes on disk
I20240502 08:00:37.554598 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: UndoDeltaBlockGCOp(f1fa312fe8e44d23970c5e3ba98c2c58) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":59,"lbm_reads_lt_1ms":4}
I20240502 08:00:37.556313 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling FlushMRSOp(a98f26c1f53f403f9a9f6ae994cd3caa): perf score=0.033361
I20240502 08:00:37.561506 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: FlushMRSOp(c53e8b3bd774466dbafe4994472975bc) complete. Timing: real 0.018s user 0.015s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":153,"dirs.run_cpu_time_us":141,"dirs.run_wall_time_us":4652,"drs_written":1,"lbm_read_time_us":61,"lbm_reads_lt_1ms":4,"lbm_write_time_us":539,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":129,"threads_started":2,"wal-append.queue_time_us":347}
I20240502 08:00:37.563395 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling UndoDeltaBlockGCOp(c53e8b3bd774466dbafe4994472975bc): 4840 bytes on disk
I20240502 08:00:37.563810 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: UndoDeltaBlockGCOp(c53e8b3bd774466dbafe4994472975bc) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":63,"lbm_reads_lt_1ms":4}
W20240502 08:00:37.565877 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 08:00:37.572518 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: FlushMRSOp(a98f26c1f53f403f9a9f6ae994cd3caa) complete. Timing: real 0.016s user 0.014s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":171,"dirs.run_cpu_time_us":117,"dirs.run_wall_time_us":1694,"drs_written":1,"lbm_read_time_us":50,"lbm_reads_lt_1ms":4,"lbm_write_time_us":455,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":65,"threads_started":1,"wal-append.queue_time_us":134}
I20240502 08:00:37.574383 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling UndoDeltaBlockGCOp(a98f26c1f53f403f9a9f6ae994cd3caa): 4586 bytes on disk
I20240502 08:00:37.574815 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: UndoDeltaBlockGCOp(a98f26c1f53f403f9a9f6ae994cd3caa) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":69,"lbm_reads_lt_1ms":4}
W20240502 08:00:37.653815 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 08:00:37.694628 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(f1fa312fe8e44d23970c5e3ba98c2c58): perf score=0.033402
I20240502 08:00:37.705461 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(f1fa312fe8e44d23970c5e3ba98c2c58) complete. Timing: real 0.011s user 0.010s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":169,"dirs.run_cpu_time_us":178,"dirs.run_wall_time_us":1494,"drs_written":1,"lbm_read_time_us":38,"lbm_reads_lt_1ms":4,"lbm_write_time_us":384,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":142,"threads_started":2,"wal-append.queue_time_us":117}
I20240502 08:00:37.707191 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(a98f26c1f53f403f9a9f6ae994cd3caa): perf score=0.033402
W20240502 08:00:37.708782 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 08:00:37.717667 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(a98f26c1f53f403f9a9f6ae994cd3caa) complete. Timing: real 0.010s user 0.009s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":36,"dirs.run_cpu_time_us":129,"dirs.run_wall_time_us":1105,"drs_written":1,"lbm_read_time_us":28,"lbm_reads_lt_1ms":4,"lbm_write_time_us":448,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":55,"threads_started":1,"wal-append.queue_time_us":130}
I20240502 08:00:37.719386 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(f1fa312fe8e44d23970c5e3ba98c2c58): 4925 bytes on disk
I20240502 08:00:37.719632 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(f1fa312fe8e44d23970c5e3ba98c2c58) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":40,"lbm_reads_lt_1ms":4}
I20240502 08:00:37.720800 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(a98f26c1f53f403f9a9f6ae994cd3caa): 4586 bytes on disk
I20240502 08:00:37.721046 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(a98f26c1f53f403f9a9f6ae994cd3caa) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":37,"lbm_reads_lt_1ms":4}
W20240502 08:00:37.736570 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:37.841730 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:37.841907 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:38.046676 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:00:38.065145 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:00:38.185489 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:38.223173 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:38.226547 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:38.275048 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 08:00:38.330230 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling FlushMRSOp(44f7d6f2a16d4c6b88526760cacc97a5): perf score=0.033334
I20240502 08:00:38.343119 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: FlushMRSOp(44f7d6f2a16d4c6b88526760cacc97a5) complete. Timing: real 0.013s user 0.011s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":254,"dirs.run_cpu_time_us":147,"dirs.run_wall_time_us":1717,"drs_written":1,"lbm_read_time_us":63,"lbm_reads_lt_1ms":4,"lbm_write_time_us":580,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":103,"threads_started":2,"wal-append.queue_time_us":90}
I20240502 08:00:38.344938 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling UndoDeltaBlockGCOp(44f7d6f2a16d4c6b88526760cacc97a5): 4587 bytes on disk
I20240502 08:00:38.345500 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: UndoDeltaBlockGCOp(44f7d6f2a16d4c6b88526760cacc97a5) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":268,"lbm_reads_lt_1ms":4}
W20240502 08:00:38.468389 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 08:00:38.473311 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling FlushMRSOp(539cf5d4596f481cb03ab4d13a19198c): perf score=0.033374
I20240502 08:00:38.477262 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(539cf5d4596f481cb03ab4d13a19198c): perf score=0.033374
I20240502 08:00:38.486689 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: FlushMRSOp(539cf5d4596f481cb03ab4d13a19198c) complete. Timing: real 0.013s user 0.011s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":135,"dirs.run_cpu_time_us":142,"dirs.run_wall_time_us":1446,"drs_written":1,"lbm_read_time_us":56,"lbm_reads_lt_1ms":4,"lbm_write_time_us":436,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":147,"threads_started":2,"wal-append.queue_time_us":137}
I20240502 08:00:38.488272 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling UndoDeltaBlockGCOp(539cf5d4596f481cb03ab4d13a19198c): 4840 bytes on disk
I20240502 08:00:38.488564 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: UndoDeltaBlockGCOp(539cf5d4596f481cb03ab4d13a19198c) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":46,"lbm_reads_lt_1ms":4}
I20240502 08:00:38.490753 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(539cf5d4596f481cb03ab4d13a19198c) complete. Timing: real 0.013s user 0.010s sys 0.002s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":254,"dirs.run_cpu_time_us":147,"dirs.run_wall_time_us":1071,"drs_written":1,"lbm_read_time_us":30,"lbm_reads_lt_1ms":4,"lbm_write_time_us":444,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":164,"threads_started":2,"wal-append.queue_time_us":140}
I20240502 08:00:38.492565 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(44f7d6f2a16d4c6b88526760cacc97a5): perf score=0.033376
I20240502 08:00:38.507277 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(44f7d6f2a16d4c6b88526760cacc97a5) complete. Timing: real 0.015s user 0.014s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":39,"dirs.run_cpu_time_us":165,"dirs.run_wall_time_us":1194,"drs_written":1,"lbm_read_time_us":34,"lbm_reads_lt_1ms":4,"lbm_write_time_us":442,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":87,"threads_started":1,"wal-append.queue_time_us":140}
I20240502 08:00:38.509011 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(539cf5d4596f481cb03ab4d13a19198c): 4840 bytes on disk
I20240502 08:00:38.509363 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(539cf5d4596f481cb03ab4d13a19198c) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":67,"lbm_reads_lt_1ms":4}
I20240502 08:00:38.510659 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(44f7d6f2a16d4c6b88526760cacc97a5): 4587 bytes on disk
I20240502 08:00:38.511049 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(44f7d6f2a16d4c6b88526760cacc97a5) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":87,"lbm_reads_lt_1ms":4}
I20240502 08:00:38.572062 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling FlushMRSOp(9e5fe86b44744df9b2e4d430fdec716f): perf score=0.033402
I20240502 08:00:38.587087 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: FlushMRSOp(9e5fe86b44744df9b2e4d430fdec716f) complete. Timing: real 0.015s user 0.013s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":144,"dirs.run_cpu_time_us":192,"dirs.run_wall_time_us":3801,"drs_written":1,"lbm_read_time_us":65,"lbm_reads_lt_1ms":4,"lbm_write_time_us":490,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":154,"threads_started":2,"wal-append.queue_time_us":153}
I20240502 08:00:38.588764 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling FlushMRSOp(44f7d6f2a16d4c6b88526760cacc97a5): perf score=0.033406
I20240502 08:00:38.599699 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: FlushMRSOp(44f7d6f2a16d4c6b88526760cacc97a5) complete. Timing: real 0.011s user 0.010s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":54,"dirs.run_cpu_time_us":165,"dirs.run_wall_time_us":1870,"drs_written":1,"lbm_read_time_us":29,"lbm_reads_lt_1ms":4,"lbm_write_time_us":368,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":76,"threads_started":1,"wal-append.queue_time_us":145}
I20240502 08:00:38.601176 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling UndoDeltaBlockGCOp(9e5fe86b44744df9b2e4d430fdec716f): 4925 bytes on disk
I20240502 08:00:38.601508 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: UndoDeltaBlockGCOp(9e5fe86b44744df9b2e4d430fdec716f) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":51,"lbm_reads_lt_1ms":4}
I20240502 08:00:38.602717 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling UndoDeltaBlockGCOp(44f7d6f2a16d4c6b88526760cacc97a5): 4587 bytes on disk
I20240502 08:00:38.603055 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: UndoDeltaBlockGCOp(44f7d6f2a16d4c6b88526760cacc97a5) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":42,"lbm_reads_lt_1ms":4}
W20240502 08:00:38.612525 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:38.630120 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:38.637508 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:38.669811 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:38.671386 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20240502 08:00:38.675501 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:38.728222 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:38.788758 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:38.892021 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:38.897678 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:39.045928 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:39.102002 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:39.119345 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:39.163450 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:39.164389 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:39.507669 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 08:00:39.519392 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(bd1c2abfe2a3416bb10d6c14d3fed46a): perf score=0.033336
I20240502 08:00:39.531302 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(bd1c2abfe2a3416bb10d6c14d3fed46a) complete. Timing: real 0.012s user 0.010s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":167,"dirs.run_cpu_time_us":151,"dirs.run_wall_time_us":1413,"drs_written":1,"lbm_read_time_us":40,"lbm_reads_lt_1ms":4,"lbm_write_time_us":450,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":122,"threads_started":2,"wal-append.queue_time_us":108}
I20240502 08:00:39.532914 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(4363e32404304568b94f615e4e29f883): perf score=0.033338
I20240502 08:00:39.542991 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(4363e32404304568b94f615e4e29f883) complete. Timing: real 0.010s user 0.009s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":60,"dirs.run_cpu_time_us":129,"dirs.run_wall_time_us":1220,"drs_written":1,"lbm_read_time_us":30,"lbm_reads_lt_1ms":4,"lbm_write_time_us":329,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":77,"threads_started":1,"wal-append.queue_time_us":145}
I20240502 08:00:39.544690 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(bd1c2abfe2a3416bb10d6c14d3fed46a): 4925 bytes on disk
I20240502 08:00:39.545065 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(bd1c2abfe2a3416bb10d6c14d3fed46a) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":42,"lbm_reads_lt_1ms":4}
I20240502 08:00:39.546351 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(4363e32404304568b94f615e4e29f883): 4585 bytes on disk
I20240502 08:00:39.546635 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(4363e32404304568b94f615e4e29f883) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":40,"lbm_reads_lt_1ms":4}
W20240502 08:00:39.572188 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:00:39.572896 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
I20240502 08:00:39.604616 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling FlushMRSOp(bd1c2abfe2a3416bb10d6c14d3fed46a): perf score=0.033361
I20240502 08:00:39.610057 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling FlushMRSOp(dd815609d7324cc588f02b6cb40cfbfd): perf score=0.033362
I20240502 08:00:39.619433 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: FlushMRSOp(bd1c2abfe2a3416bb10d6c14d3fed46a) complete. Timing: real 0.015s user 0.013s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":177,"dirs.run_cpu_time_us":173,"dirs.run_wall_time_us":1971,"drs_written":1,"lbm_read_time_us":49,"lbm_reads_lt_1ms":4,"lbm_write_time_us":507,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":167,"threads_started":2,"wal-append.queue_time_us":139}
I20240502 08:00:39.621351 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling UndoDeltaBlockGCOp(bd1c2abfe2a3416bb10d6c14d3fed46a): 4925 bytes on disk
I20240502 08:00:39.621912 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: UndoDeltaBlockGCOp(bd1c2abfe2a3416bb10d6c14d3fed46a) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":57,"lbm_reads_lt_1ms":4}
I20240502 08:00:39.628086 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: FlushMRSOp(dd815609d7324cc588f02b6cb40cfbfd) complete. Timing: real 0.018s user 0.014s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":145,"dirs.run_cpu_time_us":123,"dirs.run_wall_time_us":1737,"drs_written":1,"lbm_read_time_us":52,"lbm_reads_lt_1ms":4,"lbm_write_time_us":527,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":126,"threads_started":2,"wal-append.queue_time_us":123}
I20240502 08:00:39.629503 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling UndoDeltaBlockGCOp(dd815609d7324cc588f02b6cb40cfbfd): 4840 bytes on disk
I20240502 08:00:39.629756 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: UndoDeltaBlockGCOp(dd815609d7324cc588f02b6cb40cfbfd) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":41,"lbm_reads_lt_1ms":4}
W20240502 08:00:39.673000 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:39.677263 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:39.687062 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:39.706832 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:39.721433 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 08:00:39.747731 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling FlushMRSOp(bd1c2abfe2a3416bb10d6c14d3fed46a): perf score=0.033400
I20240502 08:00:39.765165 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: FlushMRSOp(bd1c2abfe2a3416bb10d6c14d3fed46a) complete. Timing: real 0.017s user 0.015s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":3779,"drs_written":1,"lbm_read_time_us":60,"lbm_reads_lt_1ms":4,"lbm_write_time_us":529,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":132,"threads_started":2,"wal-append.queue_time_us":132}
I20240502 08:00:39.767141 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling UndoDeltaBlockGCOp(bd1c2abfe2a3416bb10d6c14d3fed46a): 4925 bytes on disk
I20240502 08:00:39.768379 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: UndoDeltaBlockGCOp(bd1c2abfe2a3416bb10d6c14d3fed46a) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":63,"lbm_reads_lt_1ms":4}
I20240502 08:00:39.770056 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling FlushMRSOp(4363e32404304568b94f615e4e29f883): perf score=0.033404
I20240502 08:00:39.782860 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: FlushMRSOp(4363e32404304568b94f615e4e29f883) complete. Timing: real 0.013s user 0.008s sys 0.002s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":43,"dirs.run_cpu_time_us":158,"dirs.run_wall_time_us":1192,"drs_written":1,"lbm_read_time_us":56,"lbm_reads_lt_1ms":4,"lbm_write_time_us":410,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":73,"threads_started":1,"wal-append.queue_time_us":141}
I20240502 08:00:39.784847 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling UndoDeltaBlockGCOp(4363e32404304568b94f615e4e29f883): 4585 bytes on disk
I20240502 08:00:39.785179 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: UndoDeltaBlockGCOp(4363e32404304568b94f615e4e29f883) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4}
W20240502 08:00:39.935294 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:40.107120 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:40.206454 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:40.269534 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:40.362473 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:40.497532 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:40.606668 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:40.725404 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:40.742587 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
I20240502 08:00:40.792347 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling FlushMRSOp(023958ce03b84b2da063c13e4284e04d): perf score=0.033364
W20240502 08:00:40.801087 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20240502 08:00:40.805922 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling FlushMRSOp(fcbab4ec41a74ff8b0c564ac7a0b044e): perf score=0.033367
I20240502 08:00:40.807379 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: FlushMRSOp(023958ce03b84b2da063c13e4284e04d) complete. Timing: real 0.015s user 0.014s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":3197,"drs_written":1,"lbm_read_time_us":48,"lbm_reads_lt_1ms":4,"lbm_write_time_us":551,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":152,"threads_started":2,"wal-append.queue_time_us":224}
I20240502 08:00:40.809347 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling UndoDeltaBlockGCOp(023958ce03b84b2da063c13e4284e04d): 4925 bytes on disk
W20240502 08:00:40.810806 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 08:00:40.811219 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: UndoDeltaBlockGCOp(023958ce03b84b2da063c13e4284e04d) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":56,"lbm_reads_lt_1ms":4}
I20240502 08:00:40.812819 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling FlushMRSOp(a8dcc36740be4ca3ac1d43591f7549bd): perf score=0.033368
I20240502 08:00:40.820106 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: FlushMRSOp(fcbab4ec41a74ff8b0c564ac7a0b044e) complete. Timing: real 0.014s user 0.011s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":203,"dirs.run_cpu_time_us":154,"dirs.run_wall_time_us":1206,"drs_written":1,"lbm_read_time_us":33,"lbm_reads_lt_1ms":4,"lbm_write_time_us":461,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":173,"threads_started":2,"wal-append.queue_time_us":122}
I20240502 08:00:40.822227 22895 maintenance_manager.cc:419] P 7db4178eed174225bfb24f5f9445ce88: Scheduling UndoDeltaBlockGCOp(fcbab4ec41a74ff8b0c564ac7a0b044e): 4586 bytes on disk
I20240502 08:00:40.822697 22785 maintenance_manager.cc:643] P 7db4178eed174225bfb24f5f9445ce88: UndoDeltaBlockGCOp(fcbab4ec41a74ff8b0c564ac7a0b044e) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":56,"lbm_reads_lt_1ms":4}
I20240502 08:00:40.827709 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: FlushMRSOp(a8dcc36740be4ca3ac1d43591f7549bd) complete. Timing: real 0.014s user 0.013s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":46,"dirs.run_cpu_time_us":171,"dirs.run_wall_time_us":1185,"drs_written":1,"lbm_read_time_us":33,"lbm_reads_lt_1ms":4,"lbm_write_time_us":417,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":66,"threads_started":1,"wal-append.queue_time_us":129}
I20240502 08:00:40.829164 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling UndoDeltaBlockGCOp(a8dcc36740be4ca3ac1d43591f7549bd): 4840 bytes on disk
I20240502 08:00:40.829481 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: UndoDeltaBlockGCOp(a8dcc36740be4ca3ac1d43591f7549bd) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":57,"lbm_reads_lt_1ms":4}
I20240502 08:00:40.881407 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling FlushMRSOp(a8dcc36740be4ca3ac1d43591f7549bd): perf score=0.033390
I20240502 08:00:40.888149 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling FlushMRSOp(023958ce03b84b2da063c13e4284e04d): perf score=0.033390
I20240502 08:00:40.897794 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: FlushMRSOp(a8dcc36740be4ca3ac1d43591f7549bd) complete. Timing: real 0.016s user 0.015s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":2817,"dirs.run_cpu_time_us":192,"dirs.run_wall_time_us":1762,"drs_written":1,"lbm_read_time_us":58,"lbm_reads_lt_1ms":4,"lbm_write_time_us":538,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":106,"threads_started":2,"wal-append.queue_time_us":362}
I20240502 08:00:40.899168 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling UndoDeltaBlockGCOp(a8dcc36740be4ca3ac1d43591f7549bd): 4840 bytes on disk
I20240502 08:00:40.899519 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: UndoDeltaBlockGCOp(a8dcc36740be4ca3ac1d43591f7549bd) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":61,"lbm_reads_lt_1ms":4}
I20240502 08:00:40.904815 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: FlushMRSOp(023958ce03b84b2da063c13e4284e04d) complete. Timing: real 0.017s user 0.015s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":308,"dirs.run_cpu_time_us":119,"dirs.run_wall_time_us":1764,"drs_written":1,"lbm_read_time_us":47,"lbm_reads_lt_1ms":4,"lbm_write_time_us":511,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":115,"threads_started":2,"wal-append.queue_time_us":120}
I20240502 08:00:40.906451 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling UndoDeltaBlockGCOp(023958ce03b84b2da063c13e4284e04d): 4925 bytes on disk
I20240502 08:00:40.906754 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: UndoDeltaBlockGCOp(023958ce03b84b2da063c13e4284e04d) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":50,"lbm_reads_lt_1ms":4}
I20240502 08:00:40.908115 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling FlushMRSOp(fcbab4ec41a74ff8b0c564ac7a0b044e): perf score=0.033395
I20240502 08:00:40.927232 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: FlushMRSOp(fcbab4ec41a74ff8b0c564ac7a0b044e) complete. Timing: real 0.019s user 0.014s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":50,"dirs.run_cpu_time_us":164,"dirs.run_wall_time_us":1400,"drs_written":1,"lbm_read_time_us":42,"lbm_reads_lt_1ms":4,"lbm_write_time_us":450,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":93,"threads_started":1,"wal-append.queue_time_us":170}
I20240502 08:00:40.928983 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling UndoDeltaBlockGCOp(fcbab4ec41a74ff8b0c564ac7a0b044e): 4586 bytes on disk
I20240502 08:00:40.930341 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: UndoDeltaBlockGCOp(fcbab4ec41a74ff8b0c564ac7a0b044e) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":66,"lbm_reads_lt_1ms":4}
W20240502 08:00:41.006511 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:41.093325 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:41.103639 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:41.128199 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 08:00:41.128791 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:41.199018 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20240502 08:00:41.199498 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:41.219365 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:41.257830 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:41.304435 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:41.372810 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:41.421108 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:41.564678 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:41.589838 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 56 similar messages]
W20240502 08:00:41.595422 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:41.680191 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:41.752130 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:41.961589 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 38 similar messages]
W20240502 08:00:42.122157 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:42.122267 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:42.192270 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:42.234570 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:42.243080 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:42.256098 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:42.298947 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:42.310395 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:42.327361 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:00:42.736119 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:42.803547 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:42.836715 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:42.893951 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:43.003655 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:43.025972 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:43.095420 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:00:43.096506 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:43.140476 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:00:43.150121 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:43.347994 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:43.469136 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:43.530484 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:43.628401 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20240502 08:00:43.647866 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:43.684863 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:43.705148 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
I20240502 08:00:43.720309 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling FlushMRSOp(dd815609d7324cc588f02b6cb40cfbfd): perf score=0.033399
I20240502 08:00:43.751549 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: FlushMRSOp(dd815609d7324cc588f02b6cb40cfbfd) complete. Timing: real 0.031s user 0.028s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"compiler_manager_pool.queue_time_us":127,"dirs.queue_time_us":171,"dirs.run_cpu_time_us":215,"dirs.run_wall_time_us":1065,"drs_written":1,"lbm_read_time_us":59,"lbm_reads_lt_1ms":4,"lbm_write_time_us":506,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":252,"threads_started":3,"wal-append.queue_time_us":157}
I20240502 08:00:43.754171 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling UndoDeltaBlockGCOp(dd815609d7324cc588f02b6cb40cfbfd): 4840 bytes on disk
I20240502 08:00:43.754683 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: UndoDeltaBlockGCOp(dd815609d7324cc588f02b6cb40cfbfd) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":67,"lbm_reads_lt_1ms":4}
W20240502 08:00:43.829773 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
I20240502 08:00:43.857596 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling FlushMRSOp(9e5fe86b44744df9b2e4d430fdec716f): perf score=0.033391
I20240502 08:00:43.886979 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: FlushMRSOp(9e5fe86b44744df9b2e4d430fdec716f) complete. Timing: real 0.029s user 0.025s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"compiler_manager_pool.queue_time_us":6209,"dirs.queue_time_us":140,"dirs.run_cpu_time_us":166,"dirs.run_wall_time_us":1034,"drs_written":1,"lbm_read_time_us":48,"lbm_reads_lt_1ms":4,"lbm_write_time_us":1479,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":232,"threads_started":3,"wal-append.queue_time_us":133}
W20240502 08:00:43.889075 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
I20240502 08:00:43.889693 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling UndoDeltaBlockGCOp(9e5fe86b44744df9b2e4d430fdec716f): 4925 bytes on disk
I20240502 08:00:43.890200 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: UndoDeltaBlockGCOp(9e5fe86b44744df9b2e4d430fdec716f) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":59,"lbm_reads_lt_1ms":4}
W20240502 08:00:43.898725 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 08:00:43.903455 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling FlushMRSOp(88c1176b415c412fb2c824f277e4d3c8): perf score=0.033393
W20240502 08:00:43.911465 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
I20240502 08:00:43.974440 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: FlushMRSOp(88c1176b415c412fb2c824f277e4d3c8) complete. Timing: real 0.071s user 0.031s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"compiler_manager_pool.queue_time_us":191,"dirs.queue_time_us":595,"dirs.run_cpu_time_us":163,"dirs.run_wall_time_us":2192,"drs_written":1,"lbm_read_time_us":65,"lbm_reads_lt_1ms":4,"lbm_write_time_us":568,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":219,"threads_started":3,"wal-append.queue_time_us":9748}
I20240502 08:00:43.977262 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling UndoDeltaBlockGCOp(88c1176b415c412fb2c824f277e4d3c8): 4925 bytes on disk
I20240502 08:00:43.979264 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: UndoDeltaBlockGCOp(88c1176b415c412fb2c824f277e4d3c8) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":60,"lbm_reads_lt_1ms":4}
I20240502 08:00:43.985965 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling FlushMRSOp(cfff02438656430686be0f03f5c0b8ab): perf score=0.033407
W20240502 08:00:43.997651 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:43.997896 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
I20240502 08:00:44.010991 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: FlushMRSOp(cfff02438656430686be0f03f5c0b8ab) complete. Timing: real 0.025s user 0.016s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":29,"dirs.run_cpu_time_us":167,"dirs.run_wall_time_us":1445,"drs_written":1,"lbm_read_time_us":47,"lbm_reads_lt_1ms":4,"lbm_write_time_us":509,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":85,"threads_started":1,"wal-append.queue_time_us":6124}
I20240502 08:00:44.013152 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling UndoDeltaBlockGCOp(cfff02438656430686be0f03f5c0b8ab): 4584 bytes on disk
I20240502 08:00:44.013761 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: UndoDeltaBlockGCOp(cfff02438656430686be0f03f5c0b8ab) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":63,"lbm_reads_lt_1ms":4}
W20240502 08:00:44.100965 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:44.122943 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:44.304458 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:44.347636 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:44.460125 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:44.537936 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:44.612498 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:44.614495 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
I20240502 08:00:44.646504 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling FlushMRSOp(9d2283e5d28347519b38e876d840608e): perf score=0.033357
I20240502 08:00:44.659787 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: FlushMRSOp(9d2283e5d28347519b38e876d840608e) complete. Timing: real 0.013s user 0.012s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":140,"dirs.run_cpu_time_us":190,"dirs.run_wall_time_us":1336,"drs_written":1,"lbm_read_time_us":37,"lbm_reads_lt_1ms":4,"lbm_write_time_us":487,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":118,"threads_started":2,"wal-append.queue_time_us":114}
I20240502 08:00:44.661442 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling UndoDeltaBlockGCOp(9d2283e5d28347519b38e876d840608e): 4840 bytes on disk
I20240502 08:00:44.661765 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: UndoDeltaBlockGCOp(9d2283e5d28347519b38e876d840608e) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":39,"lbm_reads_lt_1ms":4}
W20240502 08:00:44.743315 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:44.793138 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:44.826212 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:00:44.826742 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:45.031602 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:45.095921 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:45.169267 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:45.251708 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
I20240502 08:00:45.274313 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling FlushMRSOp(c53e8b3bd774466dbafe4994472975bc): perf score=0.033371
I20240502 08:00:45.286619 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: FlushMRSOp(c53e8b3bd774466dbafe4994472975bc) complete. Timing: real 0.012s user 0.011s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":148,"dirs.run_cpu_time_us":169,"dirs.run_wall_time_us":1481,"drs_written":1,"lbm_read_time_us":46,"lbm_reads_lt_1ms":4,"lbm_write_time_us":409,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":118,"threads_started":2,"wal-append.queue_time_us":108}
I20240502 08:00:45.288316 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling UndoDeltaBlockGCOp(c53e8b3bd774466dbafe4994472975bc): 4840 bytes on disk
I20240502 08:00:45.288599 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: UndoDeltaBlockGCOp(c53e8b3bd774466dbafe4994472975bc) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":47,"lbm_reads_lt_1ms":4}
I20240502 08:00:45.289822 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling FlushMRSOp(ca85b40df34d4e1c91a55d7dd69e1494): perf score=0.033345
I20240502 08:00:45.299865 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: FlushMRSOp(ca85b40df34d4e1c91a55d7dd69e1494) complete. Timing: real 0.010s user 0.009s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":41,"dirs.run_cpu_time_us":138,"dirs.run_wall_time_us":1409,"drs_written":1,"lbm_read_time_us":47,"lbm_reads_lt_1ms":4,"lbm_write_time_us":312,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":67,"threads_started":1,"wal-append.queue_time_us":126}
I20240502 08:00:45.301385 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling UndoDeltaBlockGCOp(ca85b40df34d4e1c91a55d7dd69e1494): 4586 bytes on disk
I20240502 08:00:45.301667 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: UndoDeltaBlockGCOp(ca85b40df34d4e1c91a55d7dd69e1494) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":39,"lbm_reads_lt_1ms":4}
W20240502 08:00:45.319371 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:45.406262 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
I20240502 08:00:45.419046 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling FlushMRSOp(e5d5f67e026344998d5fb079f532d076): perf score=0.033344
I20240502 08:00:45.437193 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: FlushMRSOp(e5d5f67e026344998d5fb079f532d076) complete. Timing: real 0.018s user 0.013s sys 0.001s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":3503,"drs_written":1,"lbm_read_time_us":72,"lbm_reads_lt_1ms":4,"lbm_write_time_us":492,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":101,"threads_started":2,"wal-append.queue_time_us":580}
I20240502 08:00:45.439886 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling UndoDeltaBlockGCOp(e5d5f67e026344998d5fb079f532d076): 4586 bytes on disk
I20240502 08:00:45.440232 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: UndoDeltaBlockGCOp(e5d5f67e026344998d5fb079f532d076) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4}
W20240502 08:00:45.455998 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
I20240502 08:00:45.517746 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling FlushMRSOp(9ca2e5b5a0d24bcc9d1b69f998a0d928): perf score=0.033378
W20240502 08:00:45.525434 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
I20240502 08:00:45.528487 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: FlushMRSOp(9ca2e5b5a0d24bcc9d1b69f998a0d928) complete. Timing: real 0.011s user 0.009s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":138,"dirs.run_cpu_time_us":162,"dirs.run_wall_time_us":1513,"drs_written":1,"lbm_read_time_us":31,"lbm_reads_lt_1ms":4,"lbm_write_time_us":435,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":115,"threads_started":2,"wal-append.queue_time_us":106}
I20240502 08:00:45.529901 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling FlushMRSOp(015ed33d8c634473997eaa60e91269a2): perf score=0.033370
I20240502 08:00:45.540606 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: FlushMRSOp(015ed33d8c634473997eaa60e91269a2) complete. Timing: real 0.011s user 0.010s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":33,"dirs.run_cpu_time_us":118,"dirs.run_wall_time_us":1179,"drs_written":1,"lbm_read_time_us":30,"lbm_reads_lt_1ms":4,"lbm_write_time_us":380,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":51,"threads_started":1,"wal-append.queue_time_us":97}
I20240502 08:00:45.542024 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling FlushMRSOp(e24e4e639b0649e3ac135ac9358a0398): perf score=0.033336
I20240502 08:00:45.551681 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: FlushMRSOp(e24e4e639b0649e3ac135ac9358a0398) complete. Timing: real 0.010s user 0.009s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":31,"dirs.run_cpu_time_us":114,"dirs.run_wall_time_us":1106,"drs_written":1,"lbm_read_time_us":49,"lbm_reads_lt_1ms":4,"lbm_write_time_us":317,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":79,"threads_started":1,"wal-append.queue_time_us":124}
I20240502 08:00:45.553078 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling UndoDeltaBlockGCOp(015ed33d8c634473997eaa60e91269a2): 4840 bytes on disk
I20240502 08:00:45.553396 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: UndoDeltaBlockGCOp(015ed33d8c634473997eaa60e91269a2) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":50,"lbm_reads_lt_1ms":4}
I20240502 08:00:45.554929 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling FlushMRSOp(abfc26b86b17464fa7cf0b13ab73861c): perf score=0.033342
I20240502 08:00:45.560747 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling UndoDeltaBlockGCOp(e24e4e639b0649e3ac135ac9358a0398): 4840 bytes on disk
I20240502 08:00:45.561036 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: UndoDeltaBlockGCOp(e24e4e639b0649e3ac135ac9358a0398) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":52,"lbm_reads_lt_1ms":4}
I20240502 08:00:45.562350 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling UndoDeltaBlockGCOp(9ca2e5b5a0d24bcc9d1b69f998a0d928): 4586 bytes on disk
I20240502 08:00:45.562637 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: UndoDeltaBlockGCOp(9ca2e5b5a0d24bcc9d1b69f998a0d928) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":48,"lbm_reads_lt_1ms":4}
I20240502 08:00:45.566259 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: FlushMRSOp(abfc26b86b17464fa7cf0b13ab73861c) complete. Timing: real 0.011s user 0.010s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":43,"dirs.run_cpu_time_us":126,"dirs.run_wall_time_us":1133,"drs_written":1,"lbm_read_time_us":37,"lbm_reads_lt_1ms":4,"lbm_write_time_us":368,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":60,"threads_started":1,"wal-append.queue_time_us":213}
I20240502 08:00:45.568027 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling UndoDeltaBlockGCOp(abfc26b86b17464fa7cf0b13ab73861c): 4840 bytes on disk
I20240502 08:00:45.568430 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: UndoDeltaBlockGCOp(abfc26b86b17464fa7cf0b13ab73861c) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":59,"lbm_reads_lt_1ms":4}
W20240502 08:00:45.724277 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:45.759574 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:45.834868 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:45.900115 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:46.071552 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:46.203683 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 83 similar messages]
W20240502 08:00:46.210052 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:46.319510 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:46.319715 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:46.333778 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:46.399590 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:46.425856 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:46.445694 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:46.529619 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:46.556916 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:46.576942 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:46.621129 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20240502 08:00:46.704629 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 62 similar messages]
W20240502 08:00:46.794487 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:46.796180 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:46.918144 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:47.007766 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:47.054627 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:00:47.055361 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:47.176091 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:47.181422 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
I20240502 08:00:47.201828 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling FlushMRSOp(4363e32404304568b94f615e4e29f883): perf score=0.033346
I20240502 08:00:47.212885 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: FlushMRSOp(4363e32404304568b94f615e4e29f883) complete. Timing: real 0.011s user 0.008s sys 0.002s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":125,"dirs.run_cpu_time_us":159,"dirs.run_wall_time_us":1717,"drs_written":1,"lbm_read_time_us":32,"lbm_reads_lt_1ms":4,"lbm_write_time_us":419,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":109,"threads_started":2,"wal-append.queue_time_us":101}
I20240502 08:00:47.214448 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling UndoDeltaBlockGCOp(4363e32404304568b94f615e4e29f883): 4585 bytes on disk
I20240502 08:00:47.214790 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: UndoDeltaBlockGCOp(4363e32404304568b94f615e4e29f883) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":46,"lbm_reads_lt_1ms":4}
I20240502 08:00:47.325318 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling FlushMRSOp(28d5d20938a4472caf029c8d1ac89b82): perf score=0.033394
I20240502 08:00:47.336854 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: FlushMRSOp(28d5d20938a4472caf029c8d1ac89b82) complete. Timing: real 0.011s user 0.011s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":134,"dirs.run_cpu_time_us":178,"dirs.run_wall_time_us":1368,"drs_written":1,"lbm_read_time_us":43,"lbm_reads_lt_1ms":4,"lbm_write_time_us":390,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":117,"threads_started":2,"wal-append.queue_time_us":112}
I20240502 08:00:47.338325 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling UndoDeltaBlockGCOp(28d5d20938a4472caf029c8d1ac89b82): 4586 bytes on disk
I20240502 08:00:47.338618 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: UndoDeltaBlockGCOp(28d5d20938a4472caf029c8d1ac89b82) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":41,"lbm_reads_lt_1ms":4}
W20240502 08:00:47.347941 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:47.359128 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:47.381631 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
I20240502 08:00:47.467358 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling FlushMRSOp(ee06ff655f9c46c1bb06af2dd539fde7): perf score=0.033381
I20240502 08:00:47.479918 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: FlushMRSOp(ee06ff655f9c46c1bb06af2dd539fde7) complete. Timing: real 0.012s user 0.011s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":37,"dirs.run_cpu_time_us":175,"dirs.run_wall_time_us":1377,"drs_written":1,"lbm_read_time_us":32,"lbm_reads_lt_1ms":4,"lbm_write_time_us":456,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":61,"threads_started":1,"wal-append.queue_time_us":130}
I20240502 08:00:47.481774 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling UndoDeltaBlockGCOp(ee06ff655f9c46c1bb06af2dd539fde7): 4840 bytes on disk
I20240502 08:00:47.482209 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: UndoDeltaBlockGCOp(ee06ff655f9c46c1bb06af2dd539fde7) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4}
W20240502 08:00:47.520150 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 42 similar messages]
W20240502 08:00:47.549077 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
I20240502 08:00:47.591147 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling FlushMRSOp(cfff02438656430686be0f03f5c0b8ab): perf score=0.033393
I20240502 08:00:47.602962 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: FlushMRSOp(cfff02438656430686be0f03f5c0b8ab) complete. Timing: real 0.012s user 0.010s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":37,"dirs.run_cpu_time_us":181,"dirs.run_wall_time_us":1360,"drs_written":1,"lbm_read_time_us":32,"lbm_reads_lt_1ms":4,"lbm_write_time_us":374,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":60,"threads_started":1,"wal-append.queue_time_us":122}
I20240502 08:00:47.604584 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling FlushMRSOp(fcbab4ec41a74ff8b0c564ac7a0b044e): perf score=0.033372
I20240502 08:00:47.615828 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: FlushMRSOp(fcbab4ec41a74ff8b0c564ac7a0b044e) complete. Timing: real 0.011s user 0.010s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":60,"dirs.run_cpu_time_us":200,"dirs.run_wall_time_us":1239,"drs_written":1,"lbm_read_time_us":48,"lbm_reads_lt_1ms":4,"lbm_write_time_us":378,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":101,"threads_started":1,"wal-append.queue_time_us":179}
I20240502 08:00:47.617516 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling UndoDeltaBlockGCOp(fcbab4ec41a74ff8b0c564ac7a0b044e): 4586 bytes on disk
I20240502 08:00:47.617820 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: UndoDeltaBlockGCOp(fcbab4ec41a74ff8b0c564ac7a0b044e) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":52,"lbm_reads_lt_1ms":4}
I20240502 08:00:47.619203 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling UndoDeltaBlockGCOp(cfff02438656430686be0f03f5c0b8ab): 4584 bytes on disk
I20240502 08:00:47.619509 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: UndoDeltaBlockGCOp(cfff02438656430686be0f03f5c0b8ab) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":51,"lbm_reads_lt_1ms":4}
W20240502 08:00:47.662237 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:47.796859 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 08:00:47.834476 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling FlushMRSOp(e25ba5cc77334a3595a4575fa98476f0): perf score=0.033398
I20240502 08:00:47.846222 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: FlushMRSOp(e25ba5cc77334a3595a4575fa98476f0) complete. Timing: real 0.012s user 0.010s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":157,"dirs.run_cpu_time_us":175,"dirs.run_wall_time_us":1250,"drs_written":1,"lbm_read_time_us":31,"lbm_reads_lt_1ms":4,"lbm_write_time_us":387,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":129,"threads_started":2,"wal-append.queue_time_us":126}
I20240502 08:00:47.847977 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling UndoDeltaBlockGCOp(e25ba5cc77334a3595a4575fa98476f0): 4925 bytes on disk
I20240502 08:00:47.848318 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: UndoDeltaBlockGCOp(e25ba5cc77334a3595a4575fa98476f0) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":49,"lbm_reads_lt_1ms":4}
W20240502 08:00:47.872956 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:48.012934 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:48.025978 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:48.060429 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:48.114293 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
I20240502 08:00:48.124670 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling FlushMRSOp(55fb5c7475b14e0eaced5b8de1792641): perf score=0.033383
I20240502 08:00:48.135788 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: FlushMRSOp(55fb5c7475b14e0eaced5b8de1792641) complete. Timing: real 0.011s user 0.010s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":148,"dirs.run_cpu_time_us":169,"dirs.run_wall_time_us":1343,"drs_written":1,"lbm_read_time_us":29,"lbm_reads_lt_1ms":4,"lbm_write_time_us":427,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":118,"threads_started":2,"wal-append.queue_time_us":121}
I20240502 08:00:48.137450 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling UndoDeltaBlockGCOp(55fb5c7475b14e0eaced5b8de1792641): 4586 bytes on disk
I20240502 08:00:48.137737 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: UndoDeltaBlockGCOp(55fb5c7475b14e0eaced5b8de1792641) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":37,"lbm_reads_lt_1ms":4}
W20240502 08:00:48.139715 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:48.144114 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
I20240502 08:00:48.239048 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling FlushMRSOp(c53e8b3bd774466dbafe4994472975bc): perf score=0.033372
W20240502 08:00:48.242190 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 08:00:48.253155 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: FlushMRSOp(c53e8b3bd774466dbafe4994472975bc) complete. Timing: real 0.014s user 0.012s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":153,"dirs.run_cpu_time_us":154,"dirs.run_wall_time_us":1440,"drs_written":1,"lbm_read_time_us":47,"lbm_reads_lt_1ms":4,"lbm_write_time_us":797,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":144,"threads_started":2,"wal-append.queue_time_us":147}
I20240502 08:00:48.255380 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling UndoDeltaBlockGCOp(c53e8b3bd774466dbafe4994472975bc): 4840 bytes on disk
I20240502 08:00:48.256820 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: UndoDeltaBlockGCOp(c53e8b3bd774466dbafe4994472975bc) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":71,"lbm_reads_lt_1ms":4}
W20240502 08:00:48.331123 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:00:48.389889 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:48.467376 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:48.577706 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:48.603500 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:48.682273 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:48.754765 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:48.784217 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:48.818300 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 08:00:48.856432 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling FlushMRSOp(dd815609d7324cc588f02b6cb40cfbfd): perf score=0.033393
I20240502 08:00:48.867995 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: FlushMRSOp(dd815609d7324cc588f02b6cb40cfbfd) complete. Timing: real 0.011s user 0.010s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":144,"dirs.run_cpu_time_us":162,"dirs.run_wall_time_us":1653,"drs_written":1,"lbm_read_time_us":38,"lbm_reads_lt_1ms":4,"lbm_write_time_us":425,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"spinlock_wait_cycles":768,"thread_start_us":134,"threads_started":2,"wal-append.queue_time_us":150}
I20240502 08:00:48.870185 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling UndoDeltaBlockGCOp(dd815609d7324cc588f02b6cb40cfbfd): 4840 bytes on disk
I20240502 08:00:48.870568 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: UndoDeltaBlockGCOp(dd815609d7324cc588f02b6cb40cfbfd) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":41,"lbm_reads_lt_1ms":4}
W20240502 08:00:48.953203 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
I20240502 08:00:49.013092 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling FlushMRSOp(ca85b40df34d4e1c91a55d7dd69e1494): perf score=0.033384
W20240502 08:00:49.018666 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
I20240502 08:00:49.025475 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: FlushMRSOp(ca85b40df34d4e1c91a55d7dd69e1494) complete. Timing: real 0.012s user 0.009s sys 0.002s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":168,"dirs.run_cpu_time_us":136,"dirs.run_wall_time_us":1323,"drs_written":1,"lbm_read_time_us":31,"lbm_reads_lt_1ms":4,"lbm_write_time_us":531,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":151,"threads_started":2,"wal-append.queue_time_us":132}
I20240502 08:00:49.027207 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling UndoDeltaBlockGCOp(ca85b40df34d4e1c91a55d7dd69e1494): 4586 bytes on disk
I20240502 08:00:49.027503 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: UndoDeltaBlockGCOp(ca85b40df34d4e1c91a55d7dd69e1494) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":43,"lbm_reads_lt_1ms":4}
W20240502 08:00:49.112216 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20240502 08:00:49.151537 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:49.222448 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
I20240502 08:00:49.280521 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling FlushMRSOp(023958ce03b84b2da063c13e4284e04d): perf score=0.033381
W20240502 08:00:49.292037 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
I20240502 08:00:49.294337 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: FlushMRSOp(023958ce03b84b2da063c13e4284e04d) complete. Timing: real 0.014s user 0.011s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":37,"dirs.run_cpu_time_us":232,"dirs.run_wall_time_us":1348,"drs_written":1,"lbm_read_time_us":52,"lbm_reads_lt_1ms":4,"lbm_write_time_us":882,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":67,"threads_started":1,"wal-append.queue_time_us":135}
I20240502 08:00:49.295964 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling UndoDeltaBlockGCOp(023958ce03b84b2da063c13e4284e04d): 4925 bytes on disk
I20240502 08:00:49.296324 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: UndoDeltaBlockGCOp(023958ce03b84b2da063c13e4284e04d) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":77,"lbm_reads_lt_1ms":4}
W20240502 08:00:49.298372 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:49.348728 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 08:00:49.375324 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling FlushMRSOp(9e5fe86b44744df9b2e4d430fdec716f): perf score=0.033342
I20240502 08:00:49.389760 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: FlushMRSOp(9e5fe86b44744df9b2e4d430fdec716f) complete. Timing: real 0.014s user 0.008s sys 0.005s Metrics: {"bytes_written":6144,"cfile_init":1,"drs_written":1,"lbm_read_time_us":43,"lbm_reads_lt_1ms":4,"lbm_write_time_us":528,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":117,"threads_started":2,"wal-append.queue_time_us":117}
I20240502 08:00:49.391726 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling UndoDeltaBlockGCOp(9e5fe86b44744df9b2e4d430fdec716f): 4925 bytes on disk
I20240502 08:00:49.392043 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: UndoDeltaBlockGCOp(9e5fe86b44744df9b2e4d430fdec716f) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":50,"lbm_reads_lt_1ms":4}
I20240502 08:00:49.397933 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling FlushMRSOp(f1fa312fe8e44d23970c5e3ba98c2c58): perf score=0.033394
W20240502 08:00:49.405812 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
I20240502 08:00:49.414793 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: FlushMRSOp(f1fa312fe8e44d23970c5e3ba98c2c58) complete. Timing: real 0.017s user 0.014s sys 0.000s Metrics: {"bytes_written":6144,"cfile_init":1,"dirs.queue_time_us":210,"dirs.run_cpu_time_us":182,"dirs.run_wall_time_us":1368,"drs_written":1,"lbm_read_time_us":52,"lbm_reads_lt_1ms":4,"lbm_write_time_us":443,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":689,"thread_start_us":108,"threads_started":2,"wal-append.queue_time_us":170}
I20240502 08:00:49.416433 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling UndoDeltaBlockGCOp(f1fa312fe8e44d23970c5e3ba98c2c58): 4925 bytes on disk
I20240502 08:00:49.416774 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: UndoDeltaBlockGCOp(f1fa312fe8e44d23970c5e3ba98c2c58) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":61,"lbm_reads_lt_1ms":4}
W20240502 08:00:49.447233 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
I20240502 08:00:49.645608 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling FlushMRSOp(a8dcc36740be4ca3ac1d43591f7549bd): perf score=0.033363
I20240502 08:00:49.661302 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: FlushMRSOp(a8dcc36740be4ca3ac1d43591f7549bd) complete. Timing: real 0.016s user 0.013s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":33,"dirs.run_cpu_time_us":169,"dirs.run_wall_time_us":1380,"drs_written":1,"lbm_read_time_us":66,"lbm_reads_lt_1ms":4,"lbm_write_time_us":555,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":84,"threads_started":1,"wal-append.queue_time_us":176}
I20240502 08:00:49.664531 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling UndoDeltaBlockGCOp(a8dcc36740be4ca3ac1d43591f7549bd): 4840 bytes on disk
I20240502 08:00:49.664882 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: UndoDeltaBlockGCOp(a8dcc36740be4ca3ac1d43591f7549bd) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":44,"lbm_reads_lt_1ms":4}
W20240502 08:00:49.672588 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:49.778187 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:49.823402 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:49.824335 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:49.860003 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:49.913560 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
I20240502 08:00:49.919133 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling FlushMRSOp(9d2283e5d28347519b38e876d840608e): perf score=0.033363
I20240502 08:00:49.936921 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: FlushMRSOp(9d2283e5d28347519b38e876d840608e) complete. Timing: real 0.018s user 0.014s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":38,"dirs.run_cpu_time_us":145,"dirs.run_wall_time_us":1362,"drs_written":1,"lbm_read_time_us":35,"lbm_reads_lt_1ms":4,"lbm_write_time_us":542,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":62,"threads_started":1,"wal-append.queue_time_us":141}
I20240502 08:00:49.939143 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling UndoDeltaBlockGCOp(9d2283e5d28347519b38e876d840608e): 4840 bytes on disk
I20240502 08:00:49.939415 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: UndoDeltaBlockGCOp(9d2283e5d28347519b38e876d840608e) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":40,"lbm_reads_lt_1ms":4}
I20240502 08:00:50.173945 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling FlushMRSOp(539cf5d4596f481cb03ab4d13a19198c): perf score=0.033352
W20240502 08:00:50.182425 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
I20240502 08:00:50.185928 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: FlushMRSOp(539cf5d4596f481cb03ab4d13a19198c) complete. Timing: real 0.012s user 0.010s sys 0.000s Metrics: {"bytes_written":6152,"cfile_init":1,"dirs.queue_time_us":788,"dirs.run_cpu_time_us":186,"dirs.run_wall_time_us":1393,"drs_written":1,"lbm_read_time_us":33,"lbm_reads_lt_1ms":4,"lbm_write_time_us":533,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":676,"thread_start_us":108,"threads_started":2,"wal-append.queue_time_us":110}
I20240502 08:00:50.187528 25182 maintenance_manager.cc:419] P 4654231dfe994c1a9d2e37d9c3d75699: Scheduling UndoDeltaBlockGCOp(539cf5d4596f481cb03ab4d13a19198c): 4840 bytes on disk
I20240502 08:00:50.187829 25072 maintenance_manager.cc:643] P 4654231dfe994c1a9d2e37d9c3d75699: UndoDeltaBlockGCOp(539cf5d4596f481cb03ab4d13a19198c) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":38,"lbm_reads_lt_1ms":4}
W20240502 08:00:50.350184 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:50.366757 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:00:50.380530 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:50.467007 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:50.474973 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:50.532735 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:50.660174 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:50.695277 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
I20240502 08:00:50.695436 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling FlushMRSOp(e5d5f67e026344998d5fb079f532d076): perf score=0.033341
I20240502 08:00:50.705973 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: FlushMRSOp(e5d5f67e026344998d5fb079f532d076) complete. Timing: real 0.010s user 0.009s sys 0.000s Metrics: {"bytes_written":6062,"cfile_init":1,"dirs.queue_time_us":133,"dirs.run_cpu_time_us":190,"dirs.run_wall_time_us":1400,"drs_written":1,"lbm_read_time_us":52,"lbm_reads_lt_1ms":4,"lbm_write_time_us":479,"lbm_writes_lt_1ms":23,"peak_mem_usage":0,"rows_written":635,"thread_start_us":115,"threads_started":2,"wal-append.queue_time_us":131}
I20240502 08:00:50.707674 25313 maintenance_manager.cc:419] P 7eced220cf1642f3873fc385b6191e69: Scheduling UndoDeltaBlockGCOp(e5d5f67e026344998d5fb079f532d076): 4586 bytes on disk
I20240502 08:00:50.708014 25203 maintenance_manager.cc:643] P 7eced220cf1642f3873fc385b6191e69: UndoDeltaBlockGCOp(e5d5f67e026344998d5fb079f532d076) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":62,"lbm_reads_lt_1ms":4}
W20240502 08:00:50.766827 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:50.814421 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:50.947819 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:51.105628 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:51.192263 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:51.202319 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:51.216730 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:51.240551 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:51.266489 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 08:00:51.404139 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:51.409415 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:51.452126 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:51.666872 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:51.691181 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20240502 08:00:51.719017 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 08:00:51.719494 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:51.773715 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:51.848229 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:51.921046 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:51.984887 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:52.088128 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:00:52.088763 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:52.097971 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:52.296476 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:52.401139 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:52.448220 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:52.456969 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:52.466228 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:52.640640 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:00:52.717509 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:52.729063 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:52.761122 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:52.837983 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:52.952004 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:53.107375 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:53.161607 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:53.245048 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:53.260486 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:53.307025 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:53.333132 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:00:53.467587 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:53.478298 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:00:53.478986 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:53.492627 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:53.567901 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:53.618340 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:53.668219 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:53.704177 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:53.736302 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:53.983111 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:54.127671 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:54.153843 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20240502 08:00:54.249604 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:54.264248 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:54.269696 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:54.289535 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:54.311787 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:54.503417 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:54.674407 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:54.683919 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:54.841281 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:54.929821 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:55.120476 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:55.178134 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:55.187186 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:55.238727 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:55.280999 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:55.313338 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:55.366950 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:55.377990 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:55.506886 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:55.648902 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:55.700016 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:55.707948 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:55.808130 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:55.840467 22430 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 3 similar messages]
W20240502 08:00:55.841181 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 5487e66cb4d8416094a1321cb989f018: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:55.876442 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:00:55.908178 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:55.922657 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:56.003983 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:56.072036 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:56.167632 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:56.176704 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:56.313829 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 78 similar messages]
W20240502 08:00:56.335987 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:56.394156 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:56.484819 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:56.640684 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:56.681180 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:56.734221 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 61 similar messages]
W20240502 08:00:56.766189 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20240502 08:00:56.805230 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:56.822880 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:56.924114 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:56.994881 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:57.069612 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:57.206391 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:57.287458 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 08:00:57.402325 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:57.527670 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:57.637048 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:57.658069 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 08:00:57.746361 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_DELETED RPC failed for tablet 41284d865ebb4cd18f2ce08f66343262: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:57.778999 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:57.783186 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:57.808065 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:57.824517 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:57.874526 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:57.956142 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:58.051203 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:00:58.051990 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:58.072430 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:58.180806 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:58.284008 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:58.358351 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:58.371807 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:58.417052 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:58.488582 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20240502 08:00:58.524849 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:58.561200 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:00:58.690613 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:58.764143 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:00:58.764784 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:58.866189 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:59.028033 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:00:59.032150 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:59.075582 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:00:59.378505 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20240502 08:00:59.421324 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:59.458490 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:00:59.498061 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:59.505338 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:00:59.546556 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:00:59.598814 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:00:59.716174 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:00:59.858989 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:00.084818 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:01:00.245332 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:00.273591 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:00.334046 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:00.363767 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:00.388460 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:00.392838 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:00.393306 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:00.404780 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:00.470104 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:00.590978 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:00.641048 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:01:00.667877 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:00.788393 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:01:00.795732 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:00.877162 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:00.972424 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:01.001292 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:01.013471 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:01:01.286839 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:01.350967 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 08:01:01.414417 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:01.449321 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:01.461398 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:01.627794 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:01.680256 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:01.808696 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:01.889690 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20240502 08:01:01.929662 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 59 similar messages]
W20240502 08:01:01.935756 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:02.055054 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:02.061352 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:02.083774 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:02.096416 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:02.222448 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:02.374212 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:01:02.507740 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:02.534389 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:02.576993 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:02.577374 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:02.660537 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:01:02.841238 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:02.862438 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:02.877692 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:02.904212 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:03.088315 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:03.092913 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:03.115192 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:03.118140 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:03.128734 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:03.141418 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:03.224858 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:03.339332 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:03.396776 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:03.583240 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:03.594785 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:03.737929 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:01:03.782006 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:03.806625 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:01:03.916380 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:03.919313 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:04.055809 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:04.113420 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:04.208344 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:04.209378 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:04.445137 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20240502 08:01:04.461210 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:04.573195 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:04.584034 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:04.595495 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:04.614109 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:04.660802 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:04.852880 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:04.889950 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:05.013283 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:05.029525 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:05.298779 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:05.316315 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:05.350607 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:05.368078 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:05.373239 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:05.420317 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:05.557296 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:05.613263 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:05.647169 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:05.653604 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:05.734303 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:05.858076 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:05.921159 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:05.927455 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:05.998137 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:06.132146 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:06.184110 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:01:06.397781 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 08:01:06.430492 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:06.432600 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:06.470453 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:06.497093 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:06.814224 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:06.819547 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:06.892964 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20240502 08:01:06.950575 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:07.056808 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 59 similar messages]
W20240502 08:01:07.060243 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:07.099303 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:07.104030 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:07.185567 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:07.285509 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:07.369220 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:07.370159 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:07.400804 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:07.447652 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:01:07.542845 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:07.792524 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:07.805951 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 38 similar messages]
W20240502 08:01:07.814888 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:07.863126 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:07.867967 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:07.927484 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:08.033548 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:08.076622 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:08.210142 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:08.224845 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:08.225859 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:08.253012 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:08.253566 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:08.496606 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:08.629241 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:08.673555 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:08.684414 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:08.819337 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:08.827710 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:01:08.848420 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:08.855031 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20240502 08:01:08.855620 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:08.985246 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:09.201834 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:09.354290 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:09.440387 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:09.490154 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:09.502377 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20240502 08:01:09.516366 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:09.593782 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:09.849614 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:09.863029 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:09.912144 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:09.921315 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:10.059459 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:10.173828 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:10.213864 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:10.324574 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:10.332358 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:10.433753 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:10.464637 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:10.540359 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:10.596540 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:10.645637 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:10.653365 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:10.770606 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:10.860306 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:10.917224 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:10.941248 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:11.148564 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:11.215939 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:11.232568 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:11.303349 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:11.429221 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:11.430529 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:11.430893 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 08:01:11.535449 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 8 similar messages]
W20240502 08:01:11.591264 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:11.796000 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:11.879406 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20240502 08:01:11.956594 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:11.964624 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:11.993698 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:12.041874 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:12.061707 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:12.101526 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 59 similar messages]
W20240502 08:01:12.261788 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:12.358415 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:12.360638 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:12.489498 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:12.525206 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:01:12.589187 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:12.663537 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:12.694586 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:12.861407 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 08:01:12.975626 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:12.981745 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:12.987076 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:12.992416 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:13.098661 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:13.122742 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:13.225631 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:13.233572 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:13.237620 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:13.364982 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:13.523977 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:13.657518 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:13.677901 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:13.683033 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:13.725132 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:13.908494 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20240502 08:01:13.913450 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:01:13.921034 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:13.943089 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:13.999080 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:14.149394 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:14.244535 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:14.293516 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20240502 08:01:14.352732 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:14.511896 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:14.551779 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:14.573870 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:14.673113 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:14.775045 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:14.826076 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:14.877585 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:14.928702 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:15.054191 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:15.132242 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:15.138468 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:15.215874 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:15.408151 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:15.421015 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:15.447485 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:15.603883 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:15.719501 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:15.779192 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:15.823796 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:15.893680 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:15.978938 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:16.011716 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:16.042287 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:16.338009 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:16.345975 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:16.446558 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:16.489176 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:16.497457 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 08:01:16.537168 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:16.553189 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:16.608611 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:16.700908 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20240502 08:01:16.851310 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:01:16.854350 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:16.877964 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:16.957553 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:17.030567 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:17.063089 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:17.161932 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:17.239912 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:17.257472 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 08:01:17.258121 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:17.304545 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:17.381850 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:17.478235 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:17.496814 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:17.554100 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:01:17.608816 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:17.652714 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:17.758143 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:17.946496 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:17.948225 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:01:17.951961 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:17.965606 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:18.228308 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:18.286636 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:18.413049 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:18.422086 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:18.444301 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:18.521095 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:18.536947 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:18.553992 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:18.822170 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:18.888525 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:18.938781 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:18.941265 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20240502 08:01:18.969128 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:01:18.989275 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:01:19.073776 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:19.116695 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:19.176882 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:19.274097 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:19.389400 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:19.391687 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:19.500241 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:19.500454 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:19.620893 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:19.642529 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:19.756400 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:19.840816 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:19.870548 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:20.015241 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:20.032757 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:20.113636 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:20.259454 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:20.348198 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:20.371364 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:20.377203 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:20.507215 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:20.794768 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:20.858865 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:20.871757 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:20.951246 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:20.973208 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:20.977872 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:20.980157 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:21.153142 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:21.185921 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:21.192945 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:21.254523 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:21.365015 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:21.483158 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20240502 08:01:21.544118 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 08:01:21.576206 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:21.764232 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:21.767191 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:21.845392 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:21.869648 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:01:22.016520 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:22.017822 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:22.058789 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:22.097538 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:22.218967 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:22.304278 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:22.307405 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 61 similar messages]
W20240502 08:01:22.348610 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:22.360258 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:22.606995 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 38 similar messages]
W20240502 08:01:22.628393 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:22.692183 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:22.721470 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:22.739967 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:22.758749 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:22.791656 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:22.985805 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:23.112056 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:23.139866 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:23.150754 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:23.275028 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:23.328761 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 42 similar messages]
W20240502 08:01:23.355990 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:23.374876 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:23.536731 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:23.582180 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:23.622821 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:23.647266 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:23.712056 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:23.751341 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:23.838058 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:23.920785 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20240502 08:01:23.955075 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:24.008862 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:24.016002 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:01:24.096242 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:24.237025 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:01:24.267189 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:24.456070 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:24.482148 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:24.522150 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:24.525187 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:24.680004 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:24.681165 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:24.754498 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:24.755538 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:24.974417 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:25.073859 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:25.108125 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:25.266055 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:25.436116 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:25.533028 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:25.579612 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:25.607643 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:25.608839 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:25.696704 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:25.706460 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:25.818610 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:25.860385 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:25.969769 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:26.068466 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:26.091300 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:26.144757 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:26.192198 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:26.324390 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:26.379765 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:26.527045 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:26.534420 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20240502 08:01:26.536988 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:26.560266 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 08:01:26.565805 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:26.672317 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:26.944556 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:26.994028 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:27.024026 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:27.033201 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:01:27.103590 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:27.149329 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:27.168020 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:27.180296 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:27.264972 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:27.357308 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 08:01:27.485081 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:27.555343 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:27.620303 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:01:27.620838 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:27.745918 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:27.813169 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:27.856150 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:28.008828 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:28.039479 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:28.051623 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:28.156024 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:28.292450 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:28.339825 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:28.364168 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:28.374164 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 08:01:28.437883 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:28.544803 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:28.548326 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:28.662942 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:28.803254 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:28.881884 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20240502 08:01:28.895741 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:28.926358 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:28.954243 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:29.124748 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:29.231694 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:29.232492 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:01:29.278311 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:29.329094 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:29.360746 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:29.532411 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:29.615996 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:29.626542 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 32 similar messages]
W20240502 08:01:29.794137 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:29.825727 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:29.925349 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:29.953362 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:30.019662 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:30.072818 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:30.105494 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:30.303117 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:30.384742 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:30.447887 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:30.556375 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:30.626715 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:30.695380 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:30.740056 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:30.837729 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:30.869954 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:30.928550 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:30.963029 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:31.036562 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:31.107643 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:31.133898 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:31.152318 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:31.170990 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20240502 08:01:31.353586 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:31.368115 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:31.499426 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:31.508409 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:31.565006 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 08:01:31.723220 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:31.762617 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:31.774452 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:31.944993 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:31.985826 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:32.176168 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:32.202085 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:32.223539 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:32.336200 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:32.375697 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:32.395721 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:01:32.424290 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:32.511224 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 61 similar messages]
W20240502 08:01:32.511890 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:32.577176 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:32.726929 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:01:32.727777 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:32.897578 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:32.930958 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:32.971915 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:33.002400 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:33.146737 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:33.311841 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:33.341316 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:33.366416 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:33.414541 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:33.427958 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:33.453101 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:01:33.459165 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:33.472712 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:33.512166 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:33.563763 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:33.685067 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20240502 08:01:33.772267 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:33.967844 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:33.969364 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:33.992038 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:34.111658 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:34.181100 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:34.286401 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:01:34.287041 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:34.308198 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:34.572047 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:34.573416 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:34.701848 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:34.714977 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:01:34.810460 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:34.822800 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:34.972259 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:35.068919 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:35.095074 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:35.155524 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:35.234591 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:35.539026 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:35.566416 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:35.568536 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:35.575991 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:35.677628 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:35.707340 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:35.757537 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:35.785835 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:35.803273 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:35.808889 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:35.953032 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:36.068466 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20240502 08:01:36.095666 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:36.118821 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:36.176676 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:36.282475 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:36.296001 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:36.380175 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:36.566684 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 83 similar messages]
W20240502 08:01:36.567209 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:36.638218 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:36.655874 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:36.707895 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:36.749173 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:36.857939 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:37.066078 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:37.253320 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:37.279462 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:37.371096 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:37.388603 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:37.668882 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 58 similar messages]
W20240502 08:01:37.669597 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:37.746459 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:01:37.747054 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:37.753171 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:37.796264 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:01:37.812237 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:37.919437 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:38.097352 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:38.100911 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:38.140899 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:38.210458 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:38.213351 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:38.240326 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:38.245416 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:38.363461 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:38.403903 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:38.428045 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:38.485728 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 08:01:38.580471 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20240502 08:01:38.582456 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:38.623328 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:38.734936 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:38.762048 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:38.770236 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:38.916378 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:39.036222 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:39.090893 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:39.289590 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:39.330404 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:39.334537 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:01:39.347868 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:39.348563 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:39.503489 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:39.640508 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:39.801101 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:39.871872 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:39.938742 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:40.024085 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:01:40.213038 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:40.218353 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:40.304700 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:40.367249 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:40.407627 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:40.438170 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:40.549039 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:40.710690 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:40.808856 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:40.933876 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:40.945518 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:40.948238 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:41.004302 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:41.056154 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:41.116370 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:41.124341 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:41.162334 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:41.179762 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:41.195319 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:41.230852 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20240502 08:01:41.283010 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:41.378850 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:41.390637 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:41.566841 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 08:01:41.623145 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:41.639489 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:41.676798 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:41.781927 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:41.915797 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:41.945118 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:41.979900 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:42.123587 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:42.367379 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:42.411463 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:42.488030 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:42.522159 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:42.705071 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 59 similar messages]
W20240502 08:01:42.801384 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:42.866210 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:42.996145 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:42.998369 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:01:43.038694 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 42 similar messages]
W20240502 08:01:43.127244 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:43.132270 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:43.175349 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:43.185835 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:43.208258 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:43.326776 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:43.385442 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:43.402774 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:43.501936 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:01:43.551851 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:43.568540 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:43.692534 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:43.716974 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:43.748000 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:43.758071 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20240502 08:01:43.783960 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:43.864588 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:43.939950 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:44.035461 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:44.044389 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:44.112516 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:44.349756 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:44.419653 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:44.441423 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:44.477157 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:44.563215 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:44.573913 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:44.580595 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:01:44.852254 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:44.862362 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:45.076568 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:45.149838 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:45.171757 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:45.313310 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:01:45.503723 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:45.701768 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:45.720679 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:45.776094 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:45.806643 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:45.837545 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:45.843614 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:45.850705 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:45.878659 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:45.962524 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:46.092694 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:46.100154 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:46.190706 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:46.247929 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:46.266664 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:46.282971 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:46.327636 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:46.359863 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:46.404708 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:46.482834 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20240502 08:01:46.620970 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:46.652252 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 78 similar messages]
W20240502 08:01:46.829731 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:46.853900 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:46.876830 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:46.916150 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:47.086175 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:47.323966 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:47.334928 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:47.349460 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:47.365628 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:47.371062 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:47.711194 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 58 similar messages]
W20240502 08:01:47.806931 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:47.855427 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:47.931507 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:48.057051 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:48.108595 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:48.226437 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:01:48.238131 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:48.242266 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:48.246795 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:01:48.318756 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:48.379923 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:48.433032 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:48.546522 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:48.657306 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:48.688679 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:48.713917 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:48.725773 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:01:48.786113 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:48.788865 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:48.837078 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:48.841954 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:48.904728 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:48.935374 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:48.939581 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:49.012977 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20240502 08:01:49.052928 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:49.196211 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:49.341930 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:49.458331 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:49.494590 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:49.511868 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:49.614764 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:01:49.860427 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:49.930346 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:49.963588 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:49.976966 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:50.022084 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:50.114526 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:50.151245 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:50.479724 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:01:50.614881 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:50.656520 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:50.685565 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:50.700546 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:50.758855 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:50.791586 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:50.931430 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:51.023682 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:51.081946 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:51.101130 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:51.158345 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:51.158432 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:51.233762 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:51.243508 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:51.251068 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:51.288868 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:51.290319 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:51.585458 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:51.589469 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:51.678151 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 08:01:51.679772 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20240502 08:01:51.703780 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:51.760358 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:51.817837 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:52.049762 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:52.151973 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:52.262660 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:52.347772 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:52.435041 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:52.615773 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:52.617352 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:52.632812 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:52.699198 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:52.700712 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:52.765874 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 61 similar messages]
W20240502 08:01:53.048548 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:53.096247 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:53.119809 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:53.165223 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:53.253059 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:53.347736 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 08:01:53.460562 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:53.492101 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:53.571346 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:53.625543 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:53.640434 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:53.669579 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:01:53.670285 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:53.677330 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:53.679383 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:53.700939 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:53.728986 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:53.743706 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:53.884507 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:53.986807 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 42 similar messages]
W20240502 08:01:54.014856 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:54.164295 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:54.183202 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:54.234421 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:54.236366 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:54.465960 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20240502 08:01:54.513672 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:54.560575 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:54.618009 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:01:54.740737 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:54.855790 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:55.001560 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:55.028733 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:55.057319 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:55.179623 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:55.276211 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:55.324576 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:55.522665 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:55.537904 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:01:55.538803 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:55.586961 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:55.719921 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:55.771207 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:55.778491 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:55.801249 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:55.880376 22430 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 3 similar messages]
W20240502 08:01:55.881321 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 5487e66cb4d8416094a1321cb989f018: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:56.103385 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:56.161623 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:56.173794 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:56.184689 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:56.206156 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:56.212839 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:56.261052 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:56.339111 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:56.440610 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:56.484117 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:56.499480 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:56.518036 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:56.694243 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 78 similar messages]
W20240502 08:01:56.695276 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20240502 08:01:56.750574 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:56.771739 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:56.874658 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:57.094910 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:57.126950 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:57.132889 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:57.227397 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:57.596565 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:57.685344 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:57.689597 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:57.692988 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:57.775768 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_DELETED RPC failed for tablet 41284d865ebb4cd18f2ce08f66343262: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:57.798447 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:57.815944 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:01:57.914762 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 59 similar messages]
W20240502 08:01:58.007987 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:58.016099 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:01:58.085023 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:58.098577 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:01:58.153192 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:58.189177 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:58.392767 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:58.468097 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 38 similar messages]
W20240502 08:01:58.534916 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:58.541128 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:58.658001 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:58.701694 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:01:58.702358 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:58.715670 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:58.817656 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:58.833571 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:58.878921 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:58.957628 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:59.010107 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:59.051254 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:59.052227 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:01:59.058030 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:01:59.246734 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:01:59.251672 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:01:59.257203 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20240502 08:01:59.311514 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:59.516968 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:01:59.550408 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:01:59.774881 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:59.804122 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:01:59.848726 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:01:59.978763 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:01:59.996891 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:02:00.042263 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:00.060578 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:00.085775 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:00.312901 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:00.355829 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:02:00.388880 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:00.486939 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:00.531038 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:00.543128 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:00.553233 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:02:00.793913 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:00.826224 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:00.860360 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:00.995565 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:01.041309 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:01.111853 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:02:01.184237 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:01.203204 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:01.208207 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:01.443953 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:01.486131 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:01.510238 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:01.525719 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20240502 08:02:01.598691 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:01.614796 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:01.761319 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 08:02:01.826525 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:01.842233 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:01.949828 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:02.048323 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:02.063382 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:02.241338 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:02.245004 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:02.280834 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:02.342561 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:02.471177 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:02.485123 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:02.571326 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:02.647272 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:02.700165 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:02.919422 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 59 similar messages]
W20240502 08:02:02.920076 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:02.939675 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:03.147444 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:03.242154 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:03.280124 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:03.361932 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:03.462985 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:03.546219 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:03.568926 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:03.617673 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:03.621085 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:03.649235 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:03.657310 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:02:03.682598 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:03.782325 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:02:03.783262 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:03.816243 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:03.977948 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:04.025549 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:04.056730 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20240502 08:02:04.074309 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:02:04.131882 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:04.273841 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:04.289021 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:04.306643 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:04.353320 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:04.682838 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:04.722337 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:04.793648 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:04.828475 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:02:04.876827 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:04.909204 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:05.003015 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:05.037565 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:05.128357 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:05.208451 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:05.211545 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:05.309576 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:05.392149 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:05.453130 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:05.575928 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:02:05.730444 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:05.769367 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:05.807691 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:05.952554 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:06.094465 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:06.113556 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:06.201422 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:06.254308 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:06.303566 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:06.357079 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:06.364112 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:06.381150 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:06.421195 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:06.423319 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:06.657698 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:06.660146 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20240502 08:02:06.694825 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:06.721909 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:06.779155 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 08:02:06.970901 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:07.037209 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:07.145615 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:07.267798 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:07.342723 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:07.355193 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:07.433400 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:07.434526 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:07.512241 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:07.535578 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:07.558864 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:07.624488 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:07.695143 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:07.721089 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:07.849694 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:07.946230 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 08:02:08.031592 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:08.351712 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:08.370370 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:08.423014 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:08.423493 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:08.425298 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:08.433388 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:08.456796 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:08.577759 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:08.648013 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:08.802783 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:08.814873 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 08:02:08.942973 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:08.991220 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:09.017050 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:02:09.017589 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:09.036044 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:09.068822 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:09.093595 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:09.111156 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:02:09.126341 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20240502 08:02:09.381546 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:09.511044 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:09.646771 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:09.668550 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:09.735303 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:09.786300 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:09.804335 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:09.833642 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:02:09.837996 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:10.064806 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:10.103331 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:10.152555 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:10.175854 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:10.225178 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:10.251034 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:10.362586 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:10.506856 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:10.627362 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:02:10.758215 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:10.762311 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:10.789521 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:10.912813 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:10.916724 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:10.984644 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:11.059945 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:11.061317 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:11.182579 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:11.203815 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:11.504963 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:11.511876 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:11.536476 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:11.549104 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:11.573568 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:11.643317 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:11.726954 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20240502 08:02:11.811630 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 08:02:11.949029 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:12.074501 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:12.076088 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:12.117323 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:12.215824 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:12.219810 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:12.253530 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:12.444867 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:12.460196 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:12.621182 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:12.654541 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:12.724589 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:12.806494 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:12.829243 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:12.911188 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:12.957358 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 61 similar messages]
W20240502 08:02:13.086084 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:13.180759 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:13.193298 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:13.211287 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:13.346812 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:13.421921 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:13.560305 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:13.608557 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:13.620774 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:13.687320 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:13.767308 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:13.817493 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:13.830870 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:02:13.911765 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:13.936038 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:13.969408 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:14.156358 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:14.171816 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:14.192440 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:14.295177 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:02:14.302789 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20240502 08:02:14.341428 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:02:14.538265 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:14.600102 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:14.686259 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:14.726677 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:14.728086 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:14.802446 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:15.003019 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:15.020759 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:02:15.062422 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:15.062605 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:15.334779 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:15.347632 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:15.416751 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:15.433907 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:15.446795 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:15.626222 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:15.645745 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:02:15.646993 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:15.780576 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:15.790287 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:15.965473 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:15.978902 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:16.000439 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:16.129505 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:16.131922 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:16.165985 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:16.241751 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:16.313127 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:16.349123 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:16.367300 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:16.500288 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:16.574910 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:16.577762 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:16.649507 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20240502 08:02:16.882704 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:16.889174 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 08:02:16.969434 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:17.174837 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:17.189189 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:17.229961 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:17.339990 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:17.448575 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:17.595225 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:17.645444 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:17.715253 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:17.841550 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:17.929991 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:17.958236 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:17.964519 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 56 similar messages]
W20240502 08:02:18.008982 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:18.027530 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:18.053565 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:18.054669 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:18.391548 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:18.450780 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:18.471349 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:18.520622 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:18.526315 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:18.654814 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:18.662022 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:18.665645 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:18.704293 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:18.722954 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:18.767449 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:18.842967 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:18.843829 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 38 similar messages]
W20240502 08:02:18.861388 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:19.020380 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:19.132347 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20240502 08:02:19.184314 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:19.428982 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 08:02:19.444696 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:19.459192 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:19.505365 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:19.602968 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:19.658044 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:02:19.908152 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:19.981148 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:19.984377 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:20.084101 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:20.124945 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:20.192912 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:02:20.266177 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:20.295753 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:20.295876 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:20.324673 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:20.434775 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:20.519140 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:20.595311 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:20.667351 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:20.676331 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:20.737468 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:02:20.959704 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:20.975957 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:20.988649 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:21.034164 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:21.100561 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:21.197467 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:21.203598 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:21.266106 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:21.343668 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:21.346285 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:21.357867 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:21.384447 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:21.458300 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20240502 08:02:21.552707 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:21.587371 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:21.740021 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:21.894249 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 08:02:22.016158 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:22.057047 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:22.313251 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:22.409773 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:22.607715 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:22.609793 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:22.682228 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:22.716925 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:22.805861 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:22.877700 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:22.968422 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:22.984422 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 08:02:23.010257 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:23.057754 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:23.115217 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:23.175043 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:23.199230 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:23.223289 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:23.309484 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:23.365420 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:23.408203 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:23.505215 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:23.589639 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:23.595655 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:23.850663 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:23.852363 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:23.874621 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:23.925801 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 38 similar messages]
W20240502 08:02:23.941550 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:23.947708 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:24.001896 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20240502 08:02:24.042539 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:24.192024 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:24.231035 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:24.447697 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:24.541666 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:24.699877 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:24.779008 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:24.786630 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:02:24.816342 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:02:24.970623 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:24.979727 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:25.069352 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:25.249043 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:25.280818 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:25.387142 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20240502 08:02:25.401955 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:25.448307 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:25.476078 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:25.491443 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:25.512323 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:25.663439 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:25.680013 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:25.702194 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:25.880160 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:25.890456 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:25.971028 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:02:25.990995 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:26.007643 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:26.099984 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:26.168309 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:26.209111 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:26.261197 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:26.364351 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20240502 08:02:26.453208 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:26.501415 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:26.508519 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:26.579806 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:26.681234 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:26.868075 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:26.951117 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:26.992755 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 77 similar messages]
W20240502 08:02:27.024180 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:27.058816 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:27.171407 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:27.411301 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:27.451149 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:27.591697 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:27.640218 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:27.672016 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:27.832055 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:27.847435 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:27.901436 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:28.025748 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:28.029755 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 58 similar messages]
W20240502 08:02:28.038594 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:28.140386 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:28.181161 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:28.191231 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:28.331522 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:28.424075 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:28.431354 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:28.473793 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:28.489189 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:28.514766 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:28.583457 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:28.599848 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:28.711566 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:28.802865 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20240502 08:02:28.807804 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:28.919730 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:28.945577 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:02:29.065526 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:29.142091 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:29.197747 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:29.368324 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:29.369395 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:29.416004 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:29.610610 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:29.688184 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:29.814049 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 38 similar messages]
W20240502 08:02:29.868333 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:29.956473 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:29.980763 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:30.005019 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:30.161465 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:02:30.174789 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:30.183408 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:30.357365 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:30.364755 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:30.506745 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:30.531482 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:30.579489 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:30.608114 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:02:30.609342 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:30.707058 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:30.773238 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:30.795281 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:30.869467 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:30.898418 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:30.932096 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:30.958807 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:31.047586 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:02:31.077575 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:31.173820 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20240502 08:02:31.211611 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:31.299125 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:31.536183 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:31.542984 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:31.577154 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:31.793471 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:31.821192 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:31.903837 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:32.116878 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 83 similar messages]
W20240502 08:02:32.146152 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:32.163517 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:32.167301 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:32.172959 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:32.235663 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:32.244163 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:32.489552 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:32.590319 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:32.622666 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:32.660308 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:32.748579 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:32.873652 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:32.897763 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:33.083899 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 59 similar messages]
W20240502 08:02:33.087508 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:33.116343 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:33.150125 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:33.196053 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:33.254279 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:33.255776 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:33.307729 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:33.358994 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:33.372002 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:33.433321 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:33.496896 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:33.557724 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:33.852859 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:33.872834 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20240502 08:02:33.936830 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:33.949746 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:33.986070 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:34.090175 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:34.199487 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:34.199659 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 43 similar messages]
W20240502 08:02:34.445065 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:34.446384 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:34.740500 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:34.793644 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:34.803221 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:34.852110 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:02:34.878209 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:34.878365 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:34.907317 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:34.967128 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:34.998303 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:35.124220 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:35.301585 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:35.377527 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:35.378265 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:35.397929 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:35.441066 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:02:35.441624 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:35.466329 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:35.537992 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:35.670781 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:35.714548 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:35.765277 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:02:35.774369 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:35.880939 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:35.922742 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:35.945757 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:35.961165 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:35.971410 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:36.093899 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:02:36.132491 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:36.228582 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:36.330191 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:36.487397 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:36.598071 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:36.630766 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:36.640363 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20240502 08:02:36.703387 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:36.916002 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:36.985163 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:37.090102 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:37.215718 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:37.228520 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:37.274910 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 82 similar messages]
W20240502 08:02:37.335752 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:37.389495 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:37.579602 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:37.646009 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:37.710896 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:37.712054 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:37.767225 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:37.782217 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:37.867359 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:37.869354 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:37.980223 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:38.074223 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:38.136862 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:38.146543 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 08:02:38.147162 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:38.255345 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:38.284337 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:38.324601 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:38.382141 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:38.398622 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:38.576144 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:38.727013 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:38.747313 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:38.810473 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:38.861698 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:39.125561 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:39.306684 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:02:39.326040 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:39.389079 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:39.450616 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20240502 08:02:39.554378 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:39.560077 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:39.704298 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:39.764639 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:39.855257 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:39.877331 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:02:39.958878 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:39.979866 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:40.009991 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:40.077215 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:40.087365 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:40.154831 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:40.190382 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:40.370743 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:40.392481 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:40.530388 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:40.582361 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:40.640216 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:40.674857 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:02:40.820828 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:40.825366 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:40.833210 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:40.876461 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:40.909446 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:40.928634 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:02:40.958748 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:40.975890 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:40.984273 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:41.113257 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:41.126647 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:41.190099 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:41.218031 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:41.225745 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:02:41.552721 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:41.829478 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:41.848505 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:41.988415 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:42.050217 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:42.084568 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20240502 08:02:42.137174 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:42.210858 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:42.272384 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:42.282521 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 08:02:42.333410 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:42.380728 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:42.395711 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:42.449882 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:42.628719 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:42.708022 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:42.713876 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:42.728145 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:42.846046 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:42.846836 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:43.053921 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:43.157568 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 58 similar messages]
W20240502 08:02:43.215965 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:43.295626 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:43.346251 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:43.364701 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:43.434844 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:43.484936 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:43.526943 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:43.536510 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:43.551666 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:43.556072 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:43.677805 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:43.691813 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:43.943692 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:44.156368 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:44.195075 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:44.243254 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:44.245687 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:44.544723 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:02:44.579902 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:44.704313 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:44.722115 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:44.728209 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:44.758484 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:44.788273 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:44.813545 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20240502 08:02:44.833985 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:44.969328 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 38 similar messages]
W20240502 08:02:45.063470 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:45.150813 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:45.175078 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:45.229660 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:45.322423 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:45.363552 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:45.365151 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:45.600538 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:45.661484 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:45.712164 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:02:45.737807 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:45.843053 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:45.851514 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:45.974184 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:45.975414 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:45.992769 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:46.003657 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:46.086381 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:46.115103 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:02:46.153802 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:46.276233 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:46.334455 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:46.562466 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:46.600159 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:46.621013 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:02:46.637455 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:46.691183 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:46.764725 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:47.075137 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:47.085420 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:47.136834 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:47.213433 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:47.283397 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 77 similar messages]
W20240502 08:02:47.343839 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:47.353472 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20240502 08:02:47.392490 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:47.446251 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:47.446552 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:47.545047 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:47.657701 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:47.754740 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:47.755250 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:47.858095 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:47.963327 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:47.966797 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:48.162761 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:48.226091 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:48.228256 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:48.230667 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 59 similar messages]
W20240502 08:02:48.237859 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:48.246727 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:48.277652 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:48.287178 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:48.657214 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:48.765039 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:48.880947 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:48.884117 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:49.016542 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:49.045470 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:49.156126 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:49.221259 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:49.298372 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:49.440719 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:49.534700 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:49.695365 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:49.728411 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:49.729444 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:49.753656 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 08:02:49.754489 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:49.800335 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:49.848083 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:49.938863 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:49.959965 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20240502 08:02:49.975874 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:02:50.140384 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:50.226594 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:50.263587 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:50.282634 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:50.309074 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:50.374763 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:50.482430 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:50.647956 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:50.743795 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:50.784571 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:50.807304 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:50.807614 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:02:50.844969 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:50.873636 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:51.017153 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:51.025390 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:51.151497 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:51.235697 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:02:51.258482 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:51.520579 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:51.574556 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:51.582726 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:51.618093 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:51.748224 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:51.806435 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:02:51.807030 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:51.996106 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:52.124198 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:52.136870 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:52.170341 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:52.298403 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:52.336987 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 08:02:52.415308 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:52.462776 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:52.481673 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20240502 08:02:52.494993 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:52.554987 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:52.595139 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:52.617293 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:52.653563 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:52.700809 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:52.776530 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:52.812500 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:52.842188 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:52.877081 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:52.924821 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:53.136590 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:53.256609 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 59 similar messages]
W20240502 08:02:53.257315 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:53.347119 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:53.355940 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:53.409713 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:53.433725 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:53.689631 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:53.700313 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:53.853224 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:54.102308 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:54.145395 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:54.149430 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:54.170302 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:54.293843 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:54.550237 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:54.597764 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:54.609117 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:54.616302 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:54.660046 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:54.738752 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:54.825743 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:54.994264 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20240502 08:02:55.012761 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:02:55.056269 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 42 similar messages]
W20240502 08:02:55.056840 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:55.059850 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:55.077145 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:55.158476 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:55.170768 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:55.212363 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:55.295967 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:55.327849 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:55.343318 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:55.479650 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:55.488405 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:55.572286 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:55.707341 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:55.882212 22430 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 3 similar messages]
W20240502 08:02:55.882856 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 5487e66cb4d8416094a1321cb989f018: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:55.883386 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:55.890069 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:55.902065 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:55.903357 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:55.993556 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:56.036255 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:02:56.136479 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:56.390096 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:02:56.436945 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:56.485837 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:56.494328 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:56.546343 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:56.839653 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:02:56.840250 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:56.895167 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:56.897876 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:56.926424 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:57.021988 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:57.034016 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:57.180223 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:57.375921 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 08:02:57.378463 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:57.392791 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:57.411993 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:57.426479 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20240502 08:02:57.454586 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:57.457441 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:57.549108 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:57.602937 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:57.647115 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:02:57.777211 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_DELETED RPC failed for tablet 41284d865ebb4cd18f2ce08f66343262: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:57.826859 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:57.884603 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:57.923022 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:57.958127 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:57.998310 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:58.004612 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:58.149106 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:58.343278 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:58.359591 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 08:02:58.609103 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:02:58.620980 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:58.696952 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:02:58.705590 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:02:58.722175 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:58.947795 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:58.967868 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:59.055115 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:59.094601 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:59.259307 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:59.278985 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:02:59.438899 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:02:59.621882 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:02:59.623077 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:02:59.662122 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:59.710132 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:02:59.847119 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:02:59.883064 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:02:59.886111 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20240502 08:02:59.961582 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:02:59.996249 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:00.029824 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:00.073971 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:03:00.142220 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:03:00.170697 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:00.183568 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 42 similar messages]
W20240502 08:03:00.202201 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:03:00.241068 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:00.297971 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:00.328126 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:00.489892 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:00.525715 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:03:00.539229 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:00.786408 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:01.027604 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:03:01.067999 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:01.081734 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:01.179937 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:03:01.209344 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:01.306311 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:01.307286 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:01.457335 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:03:01.523348 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:03:01.538753 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:01.569254 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:01.612577 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:01.696954 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:01.711421 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:01.797953 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:01.881685 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:02.050967 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:03:02.051538 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:02.110950 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:02.137014 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:02.298496 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:02.334270 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:02.377250 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20240502 08:03:02.477699 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:02.514775 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:02.536525 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 08:03:02.537065 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:02.672151 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:02.765674 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:02.775774 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:02.780977 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:02.957624 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:02.958025 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:02.972836 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:03.003724 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:03.075879 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:03.291550 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:03.374534 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 59 similar messages]
W20240502 08:03:03.449038 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:03.566648 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:03.590373 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:03.642524 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:03.825174 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:03.907102 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:04.002844 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:04.056752 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:04.098361 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:04.099468 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:04.147925 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:04.438819 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:04.515857 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:04.541661 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:04.569162 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:04.577862 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:04.608788 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:04.642021 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20240502 08:03:04.741235 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:04.921661 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:04.949998 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:04.967756 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:04.976752 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:05.017017 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:05.094571 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 08:03:05.139047 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:05.219532 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:05.238478 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:03:05.239038 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:05.248728 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:05.307971 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:05.431552 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:05.523775 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:05.532392 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:05.760906 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:05.841107 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:06.002360 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:06.048626 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:06.206988 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:03:06.245509 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:06.269392 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:06.361976 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:06.393163 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:06.428535 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:06.459223 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:03:06.474056 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:06.503378 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:06.722021 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:06.925478 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:06.933058 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20240502 08:03:06.948112 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:06.984308 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:07.093127 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:03:07.093698 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:07.131039 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:07.142176 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:07.264007 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:07.283564 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:07.436239 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:07.544934 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:07.547276 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:07.651343 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 81 similar messages]
W20240502 08:03:07.756204 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:07.798753 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:07.819067 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:07.864403 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:07.885013 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:07.895650 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:08.079691 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:08.099299 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:08.255497 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:08.417153 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:08.430473 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 59 similar messages]
W20240502 08:03:08.431825 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:08.625494 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:08.697579 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:08.732236 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:08.745324 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:08.768368 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:08.933738 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:09.060067 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:09.064249 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:09.165670 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:09.284387 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:09.333034 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:09.386540 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:09.397083 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:09.527503 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20240502 08:03:09.626847 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:09.710114 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:09.722517 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:09.724408 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:09.799175 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:09.851997 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:09.912689 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:10.030151 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:10.155097 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:10.159055 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:10.163475 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:03:10.253530 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:10.264402 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:03:10.295356 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:10.426527 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:10.447629 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:10.651588 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:10.701558 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:10.703320 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:10.857448 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:10.928952 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:11.091336 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:11.108390 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:11.232020 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:11.331368 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:03:11.477249 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:11.494405 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:11.523329 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20240502 08:03:11.523912 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:11.565440 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:11.575525 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:11.789295 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:11.842553 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:11.867882 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:11.975483 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:11.991161 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:12.060072 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:12.129051 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:03:12.129663 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:12.192108 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:12.241339 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20240502 08:03:12.311143 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:12.465881 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:12.473145 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:12.484000 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:12.486264 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:12.719051 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:12.755383 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 08:03:12.765908 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:12.841195 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:12.859014 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:12.873996 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:12.875629 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:13.152691 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:13.238741 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:13.354146 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:13.418537 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:13.532145 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 59 similar messages]
W20240502 08:03:13.599324 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:13.609661 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:13.682108 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:13.744326 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:13.971004 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:13.971196 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:14.032109 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:14.069531 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:14.200275 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:14.321722 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:14.326579 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:14.343361 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:14.520160 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:14.549376 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:14.568161 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:14.664443 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20240502 08:03:14.699550 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:14.766629 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:14.776911 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:14.794806 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:14.926605 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:14.991086 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:15.114892 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:15.148768 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:15.164991 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:15.170938 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:15.184289 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:03:15.339005 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:03:15.339505 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:15.405843 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:15.442281 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:15.663100 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:15.922737 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:15.950711 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:16.044808 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:16.090539 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:16.120952 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:16.242326 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:16.307868 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:16.311861 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:16.426927 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:16.485141 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:16.572880 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:16.638088 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:03:16.728716 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:16.807251 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:03:16.807859 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:16.813155 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:16.814540 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20240502 08:03:16.825229 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:16.845314 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:16.887182 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:16.940025 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:17.079478 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:17.133736 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20240502 08:03:17.134361 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:17.154247 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:17.355739 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:17.434012 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:17.477953 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:17.540511 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:17.695313 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:17.747328 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:17.769766 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 08:03:17.819809 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:17.837558 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:18.065830 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:18.074962 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:18.155643 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:18.275584 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:18.323823 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:18.475028 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:18.565505 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 61 similar messages]
W20240502 08:03:18.641657 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:18.697326 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:18.754639 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:18.790089 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:18.892912 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:18.903638 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:18.928300 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:18.977586 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:19.106863 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:19.180137 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:19.248796 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:19.257071 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:19.257406 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20240502 08:03:19.267313 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:19.371671 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:19.545919 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:19.607216 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:19.691073 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:19.794582 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:19.871605 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:19.881104 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:19.917026 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:20.086607 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:20.165437 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:20.219520 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:20.323289 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:20.327155 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 08:03:20.355203 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:03:20.444706 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:20.505261 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:20.598762 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:20.750810 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:20.831156 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:20.858804 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:21.001670 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:21.152402 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:21.161932 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:21.232448 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:21.347954 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:21.427167 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:21.449621 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:21.504945 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:21.554148 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:21.561684 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:21.618270 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:21.649958 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20240502 08:03:21.655475 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:21.759347 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:21.781512 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:21.819847 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:21.957409 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:03:22.001840 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:22.027112 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:03:22.032382 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:22.132372 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:22.217514 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:22.223459 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:22.240046 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:03:22.263341 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:22.482656 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:22.644146 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:22.660817 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:22.666714 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:22.719264 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:22.781280 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:22.795656 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 08:03:23.017010 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:23.045322 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:23.201089 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:23.218341 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:23.316730 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:23.317418 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:23.603140 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 08:03:23.691076 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:23.711863 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:23.736338 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:23.751444 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:23.779619 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:23.780728 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:24.038992 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:24.077793 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:24.104069 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:24.143456 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:24.182682 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20240502 08:03:24.198370 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:24.361336 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:24.363924 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:24.411700 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:24.440506 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:24.563150 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:24.727715 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:24.787751 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:24.820777 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:24.936770 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:24.939015 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:24.966769 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:25.025620 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:25.162504 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:25.338615 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:25.394683 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:03:25.504616 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:25.523420 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:03:25.575604 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:25.601230 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:25.659907 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:25.739429 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:25.903151 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:25.931725 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:26.123502 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:26.127669 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:26.133366 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:26.165906 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:26.364323 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:26.421408 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:26.479801 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:26.494719 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:26.578558 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:26.633846 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:26.635380 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20240502 08:03:26.669595 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:26.696895 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:26.923003 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:26.968292 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:27.129318 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:03:27.132598 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:27.180142 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:27.190230 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:27.215198 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 32 similar messages]
W20240502 08:03:27.310024 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:27.387059 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:03:27.403059 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:27.428507 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:27.515448 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:27.574760 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:27.658492 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:27.682905 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:27.691716 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:27.853497 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:27.960024 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 83 similar messages]
W20240502 08:03:27.971361 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:28.068470 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:28.080536 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:28.368739 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:28.449450 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:28.495626 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:28.514940 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:28.636636 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:28.652112 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:28.699818 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:28.752550 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 08:03:28.984675 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:29.013980 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:29.022858 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:29.047569 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:29.091252 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:29.196856 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:29.252758 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:29.329386 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20240502 08:03:29.368324 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:29.393833 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:29.494699 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:29.497540 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:29.617435 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:29.770867 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:29.794106 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:29.813592 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:29.831928 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:29.882952 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:29.996237 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:30.004632 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:30.051771 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:30.138399 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:30.255671 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:30.429394 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:30.488371 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:03:30.514886 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:30.677954 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 39 similar messages]
W20240502 08:03:30.710490 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:30.782893 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:30.880157 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:31.075323 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:31.204385 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:31.219367 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:31.303033 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:31.333952 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:31.455477 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:31.495595 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:31.556268 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:31.568104 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:31.602717 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20240502 08:03:31.626129 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:31.664886 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:31.698290 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:31.720441 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:31.864377 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:31.959220 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:32.063695 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:32.166517 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:32.175047 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:32.211280 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:32.273785 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:32.370133 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:32.392789 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:32.448529 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:32.478374 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:32.478554 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:03:32.491189 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:03:32.519191 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:03:32.519840 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:32.617122 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:32.785374 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:33.016036 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 08:03:33.064394 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:33.121340 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:33.135538 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:33.324723 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:33.531407 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:33.586596 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:33.638753 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:33.650827 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:33.844379 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:33.859815 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:33.872502 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:33.940431 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 08:03:33.966871 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:34.057475 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20240502 08:03:34.065518 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:34.119518 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:34.123502 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:34.153394 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:34.257629 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:34.329383 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:34.393038 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:34.451282 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:34.505452 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:34.570389 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:34.669178 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:34.724092 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:34.814927 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:34.846623 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:34.879482 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:34.898018 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:34.964144 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:35.159883 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:35.362851 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:35.418119 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:35.507395 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:03:35.720990 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:35.742202 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:03:35.761376 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:35.867611 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:35.901793 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:35.942797 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:35.944693 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:36.092909 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:36.271814 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:36.365738 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:36.444242 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:36.486160 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:36.563962 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:36.576500 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:36.732597 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:36.736768 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:36.739470 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:36.769126 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20240502 08:03:36.794438 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:36.801771 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:36.876780 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:36.911649 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:37.064366 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:37.109344 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:37.113843 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:37.274914 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:37.390404 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:37.410552 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:37.462694 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:37.509637 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:37.555559 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:03:37.639407 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:03:37.747787 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:37.924798 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:03:37.971419 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:38.006052 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:38.016433 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:38.127018 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 08:03:38.195330 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:38.283380 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:38.370620 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:38.415246 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:38.423372 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:38.541064 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:38.633780 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:38.650914 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:38.878482 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:38.879037 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:38.968708 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 58 similar messages]
W20240502 08:03:39.024411 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:39.034772 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:39.062366 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:39.326098 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:39.335326 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:39.408636 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:39.441960 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:39.454315 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:39.467641 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:39.500259 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20240502 08:03:39.566977 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:39.584789 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:39.590270 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:39.686342 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:39.786964 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:39.839768 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:39.845973 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:39.861421 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:39.911845 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:40.155167 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:40.329277 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:40.465304 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:40.659126 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:40.724155 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 42 similar messages]
W20240502 08:03:40.724689 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:40.781085 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 38 similar messages]
W20240502 08:03:40.801100 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:40.890049 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:40.905997 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:40.930240 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:40.994710 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:40.996090 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:41.307247 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:41.442879 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:41.500259 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:41.564430 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:41.678478 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:41.679546 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:41.693935 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:41.826967 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:41.870404 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:41.928083 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:41.931221 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:42.019856 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:42.048328 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:42.122007 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:42.209527 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:42.223713 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:42.230921 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20240502 08:03:42.233029 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:42.239609 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:42.275571 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:42.368798 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:42.375780 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:42.442704 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:42.640177 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 27 similar messages]
W20240502 08:03:42.641624 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:03:42.785033 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:42.796934 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:43.080605 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:43.118742 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:43.128472 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:43.150686 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 79 similar messages]
W20240502 08:03:43.259595 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:43.379276 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:43.395313 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:43.398228 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:43.418433 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:03:43.574584 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:43.739552 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:43.841817 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:43.849288 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:44.007213 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:44.091260 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:44.126673 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 60 similar messages]
W20240502 08:03:44.127410 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:44.166152 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:44.336870 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:44.347843 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:44.467572 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:44.505919 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:44.543017 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:44.549113 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:44.550282 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20240502 08:03:44.577908 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:44.594764 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:44.636374 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:44.673038 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:44.676600 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:44.745939 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:44.826665 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:44.928341 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:44.985644 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:45.084410 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:45.437237 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:45.472637 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:45.650288 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:45.683980 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:45.691884 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:45.768311 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:45.778028 22783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 40 similar messages]
W20240502 08:03:45.859401 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:45.869913 24807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 41 similar messages]
W20240502 08:03:45.994104 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:46.006547 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:46.067205 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:46.186156 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:46.344987 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:46.418926 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:46.510844 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:46.678172 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:46.717536 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:46.737020 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:46.778472 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:46.802088 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20240502 08:03:46.887239 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:46.946120 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:46.956097 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:46.973959 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:47.034359 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:47.170365 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:47.217196 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:47.254592 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:47.270136 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:47.365413 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:47.482241 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:47.484166 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:47.523386 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:47.616603 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:47.696985 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:47.809103 25070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20240502 08:03:47.835880 22914 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20240502 08:03:47.977761 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:47.995663 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:47.998632 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:48.103044 22652 consensus_peers.cc:485] T 539cf5d4596f481cb03ab4d13a19198c P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:48.153813 25201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 80 similar messages]
W20240502 08:03:48.248551 25070 consensus_peers.cc:485] T 4f3fb3e3b89540a6bd7c6ff623ff4ff7 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:48.269891 22652 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
W20240502 08:03:48.400048 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:48.527248 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:48.549221 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:48.607322 22521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20240502 08:03:48.639083 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
W20240502 08:03:48.681070 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
W20240502 08:03:48.750489 25201 consensus_peers.cc:485] T 7ecf0142784a4dacbb61f15215dbe24c P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:48.782914 22914 consensus_peers.cc:485] T a8dcc36740be4ca3ac1d43591f7549bd P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:48.964113 24807 consensus_peers.cc:485] T 68cfb914f8c140728ba025b37d7da7f5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
W20240502 08:03:48.975828 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
W20240502 08:03:49.192520 25070 consensus_peers.cc:485] T f1862e3480644235adaf4a58f2c45bd9 P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
W20240502 08:03:49.232888 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20240502 08:03:49.246390 22652 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 62 similar messages]
W20240502 08:03:49.283341 25201 consensus_peers.cc:485] T 5b7fbabf24b240879fe8f15bcfa4b377 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:49.365423 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20240502 08:03:49.374887 25201 consensus_peers.cc:485] T 7f8b8ce864e24201ba3d318fdce76d66 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20240502 08:03:49.400118 7072 consensus_queue.cc:579] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Leader has been unable to successfully communicate with peer 74f9aa91c2c448f980b512fe2e8d672e for more than 300 seconds (300.269s)
W20240502 08:03:49.401190 22652 consensus_peers.cc:462] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Unable to start Tablet Copy on peer: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 08:03:49.405977 22716 consensus_queue.cc:237] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 08:03:49.407135 22584 raft_consensus.cc:1261] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:03:49.407294 22847 raft_consensus.cc:1261] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:03:49.407759 7138 consensus_queue.cc:1035] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:03:49.408036 6766 consensus_queue.cc:1035] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:03:49.409698 7138 raft_consensus.cc:2932] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } }
I20240502 08:03:49.410058 22584 raft_consensus.cc:2932] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } }
I20240502 08:03:49.412154 22428 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 68f96ca3354645fd91f666109eeaf847 with cas_config_opid_index 4: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:03:49.413304 22847 raft_consensus.cc:2932] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } }
I20240502 08:03:49.413421 22438 catalog_manager.cc:5484] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 4 to 5, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } }
W20240502 08:03:49.418046 22430 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111) [suppressed 3 similar messages]
I20240502 08:03:49.418602 22716 consensus_queue.cc:237] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 08:03:49.419719 22584 raft_consensus.cc:1261] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:03:49.419844 22847 raft_consensus.cc:1261] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
W20240502 08:03:49.420033 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet 68f96ca3354645fd91f666109eeaf847: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 08:03:49.420146 6766 consensus_queue.cc:1035] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
W20240502 08:03:49.420280 22651 consensus_peers.cc:485] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): Couldn't send request to peer 02a92e1c084b4308b9e855a4cc2c295f. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 68f96ca3354645fd91f666109eeaf847. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.420284 7072 consensus_queue.cc:1035] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:03:49.421622 7138 raft_consensus.cc:2932] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 08:03:49.421962 22847 raft_consensus.cc:2932] T 68f96ca3354645fd91f666109eeaf847 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 08:03:49.422046 22584 raft_consensus.cc:2932] T 68f96ca3354645fd91f666109eeaf847 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 08:03:49.423861 22428 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 68f96ca3354645fd91f666109eeaf847 with cas_config_opid_index 5: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:03:49.425623 22438 catalog_manager.cc:5484] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 5 to 6, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:03:49.431054 6766 consensus_queue.cc:579] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Leader has been unable to successfully communicate with peer 74f9aa91c2c448f980b512fe2e8d672e for more than 300 seconds (300.035s)
I20240502 08:03:49.433514 22716 consensus_queue.cc:237] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 08:03:49.434739 24870 raft_consensus.cc:1261] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 08:03:49.434935 22652 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.435128 7138 consensus_queue.cc:1035] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 08:03:49.435277 22653 consensus_peers.cc:485] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): Couldn't send request to peer 7db4178eed174225bfb24f5f9445ce88. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c3a7b2e52ba641e993cf1d5dc9724473. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.436554 7072 raft_consensus.cc:2932] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 08:03:49.436942 24870 raft_consensus.cc:2932] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 08:03:49.437956 22428 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet c3a7b2e52ba641e993cf1d5dc9724473 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:03:49.440011 22440 catalog_manager.cc:5484] T c3a7b2e52ba641e993cf1d5dc9724473 P 6d02017318a2461796d431962fe6a537 reported cstate change: config changed from index 4 to 5, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
W20240502 08:03:49.491750 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20240502 08:03:49.528201 5698 consensus_queue.cc:579] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [LEADER]: Leader has been unable to successfully communicate with peer 74f9aa91c2c448f980b512fe2e8d672e for more than 300 seconds (300.066s)
I20240502 08:03:49.534747 24870 consensus_queue.cc:237] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
W20240502 08:03:49.536000 24805 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): Couldn't send request to peer 725969d490a04fdfb3ba890c3f5096e3. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a071d3a1489e4d97ae6ceea4b0478459. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.536036 22847 raft_consensus.cc:1261] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 08:03:49.536060 22914 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20240502 08:03:49.536368 7209 consensus_queue.cc:1035] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 08:03:49.537029 24807 consensus_peers.cc:485] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.537555 6920 consensus_queue.cc:579] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Leader has been unable to successfully communicate with peer 74f9aa91c2c448f980b512fe2e8d672e for more than 300 seconds (300.357s)
W20240502 08:03:49.538518 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20240502 08:03:49.538729 6672 raft_consensus.cc:2932] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } } }
I20240502 08:03:49.539105 22847 raft_consensus.cc:2932] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } } }
I20240502 08:03:49.540984 22428 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet a071d3a1489e4d97ae6ceea4b0478459 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:03:49.542191 22436 catalog_manager.cc:5484] T a071d3a1489e4d97ae6ceea4b0478459 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 4 to 5, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } } }
I20240502 08:03:49.545455 22584 consensus_queue.cc:237] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 08:03:49.547194 22847 raft_consensus.cc:1261] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 8bea0fe8d8044a2685f92f7a10c9cf85: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 08:03:49.547235 22519 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): Couldn't send request to peer 725969d490a04fdfb3ba890c3f5096e3. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
W20240502 08:03:49.547466 22521 consensus_peers.cc:485] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.547714 6920 consensus_queue.cc:1035] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:03:49.549095 6515 raft_consensus.cc:2932] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } } }
I20240502 08:03:49.549453 22847 raft_consensus.cc:2932] T a5374313bbba4b10904e0ea18851c07d P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } } }
I20240502 08:03:49.550505 22431 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet a5374313bbba4b10904e0ea18851c07d with cas_config_opid_index 6: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:03:49.551972 22436 catalog_manager.cc:5484] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85 reported cstate change: config changed from index 6 to 7, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New cstate: current_term: 1 leader_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:03:49.561857 5698 consensus_queue.cc:579] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [LEADER]: Leader has been unable to successfully communicate with peer 74f9aa91c2c448f980b512fe2e8d672e for more than 300 seconds (300.109s)
I20240502 08:03:49.564990 24872 consensus_queue.cc:237] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } }
I20240502 08:03:49.566499 25264 raft_consensus.cc:1261] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 6d02017318a2461796d431962fe6a537: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
W20240502 08:03:49.566536 24808 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): Couldn't send request to peer 8bea0fe8d8044a2685f92f7a10c9cf85. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7eb1b4bcf48b4cbea5cfb4dd64de2c70. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.566805 7209 consensus_queue.cc:1035] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
W20240502 08:03:49.566949 24807 consensus_peers.cc:485] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.568356 5698 raft_consensus.cc:2932] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537 [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } } }
I20240502 08:03:49.568799 25264 raft_consensus.cc:2932] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } } }
I20240502 08:03:49.569741 22428 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 7eb1b4bcf48b4cbea5cfb4dd64de2c70 with cas_config_opid_index -1: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:03:49.571713 22436 catalog_manager.cc:5484] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 7eced220cf1642f3873fc385b6191e69 reported cstate change: config changed from index -1 to 2, NON_VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) added. New cstate: current_term: 1 leader_uuid: "6d02017318a2461796d431962fe6a537" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } } }
I20240502 08:03:49.572774 6889 consensus_queue.cc:579] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Leader has been unable to successfully communicate with peer 74f9aa91c2c448f980b512fe2e8d672e for more than 300 seconds (300.276s)
I20240502 08:03:49.580471 22847 consensus_queue.cc:237] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } }
I20240502 08:03:49.581845 24872 raft_consensus.cc:1261] T 07409f4ddd0040fc98e6b2064e2df523 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
W20240502 08:03:49.581944 22784 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): Couldn't send request to peer 8bea0fe8d8044a2685f92f7a10c9cf85. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 07409f4ddd0040fc98e6b2064e2df523. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.582274 7218 consensus_queue.cc:1035] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
W20240502 08:03:49.582737 22783 consensus_peers.cc:485] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.583920 7218 raft_consensus.cc:2932] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } } }
I20240502 08:03:49.584355 24872 raft_consensus.cc:2932] T 07409f4ddd0040fc98e6b2064e2df523 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } } }
I20240502 08:03:49.585302 22431 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 07409f4ddd0040fc98e6b2064e2df523 with cas_config_opid_index -1: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:03:49.587363 22436 catalog_manager.cc:5484] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index -1 to 2, NON_VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) added. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 08:03:49.653822 25201 consensus_peers.cc:485] T 266266d3892c47d6826085bdc7fc042e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
W20240502 08:03:49.660981 25201 consensus_peers.cc:485] T fc05622956224d2ba1d829aaf1ddea3e P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20240502 08:03:49.661830 7218 consensus_queue.cc:579] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Leader has been unable to successfully communicate with peer 74f9aa91c2c448f980b512fe2e8d672e for more than 300 seconds (300.013s)
W20240502 08:03:49.662739 22783 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20240502 08:03:49.664757 22847 consensus_queue.cc:237] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } }
I20240502 08:03:49.666028 22979 raft_consensus.cc:1261] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:03:49.666026 22716 raft_consensus.cc:1261] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:03:49.666496 5321 consensus_queue.cc:1035] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:03:49.666811 5321 consensus_queue.cc:1035] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:03:49.668622 6889 raft_consensus.cc:2932] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } }
I20240502 08:03:49.669281 22716 raft_consensus.cc:2932] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } }
I20240502 08:03:49.672113 22431 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet bd1c2abfe2a3416bb10d6c14d3fed46a with cas_config_opid_index 4: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:03:49.672829 22979 raft_consensus.cc:2932] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } }
I20240502 08:03:49.673067 22436 catalog_manager.cc:5484] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 4 to 5, NON_VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } }
I20240502 08:03:49.678597 22847 consensus_queue.cc:237] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } }
I20240502 08:03:49.679931 22979 raft_consensus.cc:1261] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
W20240502 08:03:49.680395 22784 consensus_peers.cc:485] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 -> Peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451): Couldn't send request to peer 8bea0fe8d8044a2685f92f7a10c9cf85. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: bd1c2abfe2a3416bb10d6c14d3fed46a. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.680506 22716 raft_consensus.cc:1261] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:03:49.680495 7218 consensus_queue.cc:1035] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:03:49.680828 7218 consensus_queue.cc:1035] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:03:49.681844 7218 raft_consensus.cc:2932] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } } }
I20240502 08:03:49.682246 22979 raft_consensus.cc:2932] T bd1c2abfe2a3416bb10d6c14d3fed46a P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } } }
I20240502 08:03:49.683557 22431 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet bd1c2abfe2a3416bb10d6c14d3fed46a with cas_config_opid_index 5: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:03:49.683805 22716 raft_consensus.cc:2932] T bd1c2abfe2a3416bb10d6c14d3fed46a P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, NON_VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) added. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } } }
I20240502 08:03:49.685693 22438 catalog_manager.cc:5484] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 5 to 6, NON_VOTER 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129) added. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:03:49.693220 6639 consensus_queue.cc:579] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Leader has been unable to successfully communicate with peer 74f9aa91c2c448f980b512fe2e8d672e for more than 300 seconds (300.325s)
W20240502 08:03:49.694862 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20240502 08:03:49.699025 25264 consensus_queue.cc:237] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 08:03:49.700389 24872 raft_consensus.cc:1261] T 2b5aafb8ddb24e51867f856884a6ee33 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 7eced220cf1642f3873fc385b6191e69: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
W20240502 08:03:49.700423 25200 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): Couldn't send request to peer 02a92e1c084b4308b9e855a4cc2c295f. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2b5aafb8ddb24e51867f856884a6ee33. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.700752 7156 consensus_queue.cc:1035] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
W20240502 08:03:49.701126 25201 consensus_peers.cc:485] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.702211 6639 raft_consensus.cc:2932] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69 [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 08:03:49.702590 24872 raft_consensus.cc:2932] T 2b5aafb8ddb24e51867f856884a6ee33 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 08:03:49.704689 22430 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 2b5aafb8ddb24e51867f856884a6ee33 with cas_config_opid_index -1: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:03:49.705636 22440 catalog_manager.cc:5484] T 2b5aafb8ddb24e51867f856884a6ee33 P 6d02017318a2461796d431962fe6a537 reported cstate change: config changed from index -1 to 2, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New cstate: current_term: 1 leader_uuid: "7eced220cf1642f3873fc385b6191e69" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 08:03:49.725816 6074 consensus_queue.cc:579] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Leader has been unable to successfully communicate with peer 74f9aa91c2c448f980b512fe2e8d672e for more than 300 seconds (300.064s)
I20240502 08:03:49.731834 22979 consensus_queue.cc:237] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
W20240502 08:03:49.733536 22912 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387): Couldn't send request to peer 725969d490a04fdfb3ba890c3f5096e3. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4679a4f9194a4dbcb85c28e31f6ac894. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.733615 24872 raft_consensus.cc:1261] T 4679a4f9194a4dbcb85c28e31f6ac894 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 08:03:49.734194 6074 consensus_queue.cc:1035] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
W20240502 08:03:49.734762 22914 consensus_peers.cc:485] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.735960 7228 raft_consensus.cc:2932] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } } }
I20240502 08:03:49.736981 24872 raft_consensus.cc:2932] T 4679a4f9194a4dbcb85c28e31f6ac894 P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } } }
I20240502 08:03:49.737460 22429 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 4679a4f9194a4dbcb85c28e31f6ac894 with cas_config_opid_index -1: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:03:49.739182 22436 catalog_manager.cc:5484] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index -1 to 2, NON_VOTER 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:03:49.895505 6766 consensus_queue.cc:579] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Leader has been unable to successfully communicate with peer 74f9aa91c2c448f980b512fe2e8d672e for more than 300 seconds (300.442s)
W20240502 08:03:49.896404 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
I20240502 08:03:49.898495 22716 consensus_queue.cc:237] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 08:03:49.899801 22847 raft_consensus.cc:1261] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 08:03:49.899854 22651 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): Couldn't send request to peer 02a92e1c084b4308b9e855a4cc2c295f. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.899899 7137 consensus_queue.cc:579] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Leader has been unable to successfully communicate with peer 74f9aa91c2c448f980b512fe2e8d672e for more than 300 seconds (300.260s)
I20240502 08:03:49.900198 7072 consensus_queue.cc:1035] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240502 08:03:49.901378 22652 consensus_peers.cc:485] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.901971 7072 raft_consensus.cc:2932] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 08:03:49.902418 22847 raft_consensus.cc:2932] T cceaa8887081401dbc9154d265004c7b P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 08:03:49.903872 22428 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet cceaa8887081401dbc9154d265004c7b with cas_config_opid_index 6: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:03:49.905938 22436 catalog_manager.cc:5484] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 6 to 7, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:03:49.908174 25134 consensus_queue.cc:237] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 08:03:49.908844 7138 consensus_queue.cc:579] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Leader has been unable to successfully communicate with peer 74f9aa91c2c448f980b512fe2e8d672e for more than 300 seconds (300.208s)
W20240502 08:03:49.910205 25069 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425): Couldn't send request to peer 02a92e1c084b4308b9e855a4cc2c295f. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d2eebf9aaa2c4f92a4fefbe5bb6feb0c. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.910462 24872 raft_consensus.cc:1261] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Refusing update from remote peer 4654231dfe994c1a9d2e37d9c3d75699: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 08:03:49.910786 7137 consensus_queue.cc:1035] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
W20240502 08:03:49.911105 25070 consensus_peers.cc:485] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.913106 22716 consensus_queue.cc:237] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } }
I20240502 08:03:49.914229 22979 raft_consensus.cc:1261] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:03:49.914296 25134 raft_consensus.cc:1261] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:03:49.914450 5666 raft_consensus.cc:2932] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 08:03:49.914582 7138 consensus_queue.cc:1035] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:03:49.914685 7150 consensus_queue.cc:1035] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:03:49.914978 24872 raft_consensus.cc:2932] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 6d02017318a2461796d431962fe6a537 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } } }
I20240502 08:03:49.916074 7150 raft_consensus.cc:2932] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } }
I20240502 08:03:49.916283 22431 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet d2eebf9aaa2c4f92a4fefbe5bb6feb0c with cas_config_opid_index -1: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:03:49.916620 25134 raft_consensus.cc:2932] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } }
I20240502 08:03:49.917572 22428 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet da39029564d042f6acfc23f99fb63dfd with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:03:49.918738 22440 catalog_manager.cc:5484] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index -1 to 2, NON_VOTER 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132) added. New cstate: current_term: 1 leader_uuid: "4654231dfe994c1a9d2e37d9c3d75699" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:03:49.919740 22438 catalog_manager.cc:5484] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 3 to 4, VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } }
I20240502 08:03:49.920320 22979 raft_consensus.cc:2932] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } }
I20240502 08:03:49.925148 22716 consensus_queue.cc:237] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 08:03:49.926330 25134 raft_consensus.cc:1261] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 08:03:49.926637 22653 consensus_peers.cc:485] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055): Couldn't send request to peer 7db4178eed174225bfb24f5f9445ce88. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: da39029564d042f6acfc23f99fb63dfd. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.926792 7072 consensus_queue.cc:1035] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:03:49.927194 22979 raft_consensus.cc:1261] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 08:03:49.927356 25201 consensus_peers.cc:485] T 325b6a98112d4eef9488d7ef657bc375 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20240502 08:03:49.927570 7072 consensus_queue.cc:1035] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:03:49.928891 6766 raft_consensus.cc:2932] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 08:03:49.930050 22979 raft_consensus.cc:2932] T da39029564d042f6acfc23f99fb63dfd P 02a92e1c084b4308b9e855a4cc2c295f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 08:03:49.930296 22428 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet da39029564d042f6acfc23f99fb63dfd with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:03:49.930999 25134 raft_consensus.cc:2932] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 08:03:49.934109 22438 catalog_manager.cc:5484] T da39029564d042f6acfc23f99fb63dfd P 4654231dfe994c1a9d2e37d9c3d75699 reported cstate change: config changed from index 4 to 5, NON_VOTER 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131) added. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } } }
I20240502 08:03:49.959770 7150 consensus_queue.cc:579] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Leader has been unable to successfully communicate with peer 74f9aa91c2c448f980b512fe2e8d672e for more than 300 seconds (300.303s)
I20240502 08:03:49.962688 22716 consensus_queue.cc:237] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 08:03:49.963987 25264 raft_consensus.cc:1261] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Refusing update from remote peer 725969d490a04fdfb3ba890c3f5096e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
W20240502 08:03:49.964064 22650 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): Couldn't send request to peer 6d02017318a2461796d431962fe6a537. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e25ba5cc77334a3595a4575fa98476f0. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.964418 7072 consensus_queue.cc:1035] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
W20240502 08:03:49.964855 22652 consensus_peers.cc:485] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:49.965988 7150 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 08:03:49.966370 25264 raft_consensus.cc:2932] T e25ba5cc77334a3595a4575fa98476f0 P 7eced220cf1642f3873fc385b6191e69 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 08:03:49.967201 22428 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet e25ba5cc77334a3595a4575fa98476f0 with cas_config_opid_index 9: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:03:49.969472 22435 catalog_manager.cc:5484] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3 reported cstate change: config changed from index 9 to 10, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "725969d490a04fdfb3ba890c3f5096e3" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:03:49.993355 7288 ts_tablet_manager.cc:902] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: Initiating tablet copy from peer 8bea0fe8d8044a2685f92f7a10c9cf85 (127.16.85.129:37451)
I20240502 08:03:49.993888 7288 tablet_copy_client.cc:308] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.129:37451
I20240502 08:03:49.994307 22606 tablet_copy_service.cc:140] P 8bea0fe8d8044a2685f92f7a10c9cf85: Received BeginTabletCopySession request for tablet a5374313bbba4b10904e0ea18851c07d from peer 725969d490a04fdfb3ba890c3f5096e3 ({username='slave'} at 127.16.85.130:39611)
I20240502 08:03:49.994398 22606 tablet_copy_service.cc:161] P 8bea0fe8d8044a2685f92f7a10c9cf85: Beginning new tablet copy session on tablet a5374313bbba4b10904e0ea18851c07d from peer 725969d490a04fdfb3ba890c3f5096e3 at {username='slave'} at 127.16.85.130:39611: session id = 725969d490a04fdfb3ba890c3f5096e3-a5374313bbba4b10904e0ea18851c07d
I20240502 08:03:49.995498 22606 tablet_copy_source_session.cc:205] T a5374313bbba4b10904e0ea18851c07d P 8bea0fe8d8044a2685f92f7a10c9cf85: Tablet Copy: opened 5 blocks and 1 log segments
I20240502 08:03:49.996275 7288 ts_tablet_manager.cc:1875] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 08:03:49.997939 7288 ts_tablet_manager.cc:1888] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.5
I20240502 08:03:49.998083 7288 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a5374313bbba4b10904e0ea18851c07d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:03:49.998970 7288 tablet_copy_client.cc:783] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Starting download of 5 data blocks...
I20240502 08:03:50.003294 7288 tablet_copy_client.cc:647] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Starting download of 1 WAL segments...
I20240502 08:03:50.004326 7298 ts_tablet_manager.cc:902] T 68f96ca3354645fd91f666109eeaf847 P 02a92e1c084b4308b9e855a4cc2c295f: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 08:03:50.004485 7288 tablet_copy_client.cc:523] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:03:50.004906 7298 tablet_copy_client.cc:308] T 68f96ca3354645fd91f666109eeaf847 P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33387
I20240502 08:03:50.005288 22737 tablet_copy_service.cc:140] P 725969d490a04fdfb3ba890c3f5096e3: Received BeginTabletCopySession request for tablet 68f96ca3354645fd91f666109eeaf847 from peer 02a92e1c084b4308b9e855a4cc2c295f ({username='slave'} at 127.16.85.132:44073)
I20240502 08:03:50.005368 22737 tablet_copy_service.cc:161] P 725969d490a04fdfb3ba890c3f5096e3: Beginning new tablet copy session on tablet 68f96ca3354645fd91f666109eeaf847 from peer 02a92e1c084b4308b9e855a4cc2c295f at {username='slave'} at 127.16.85.132:44073: session id = 02a92e1c084b4308b9e855a4cc2c295f-68f96ca3354645fd91f666109eeaf847
I20240502 08:03:50.006505 22737 tablet_copy_source_session.cc:205] T 68f96ca3354645fd91f666109eeaf847 P 725969d490a04fdfb3ba890c3f5096e3: Tablet Copy: opened 5 blocks and 1 log segments
I20240502 08:03:50.006557 7288 tablet_bootstrap.cc:492] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 08:03:50.007400 7298 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 68f96ca3354645fd91f666109eeaf847. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:03:50.009097 7298 tablet_copy_client.cc:783] T 68f96ca3354645fd91f666109eeaf847 P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Starting download of 5 data blocks...
I20240502 08:03:50.014725 7298 tablet_copy_client.cc:647] T 68f96ca3354645fd91f666109eeaf847 P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Starting download of 1 WAL segments...
I20240502 08:03:50.015381 7288 tablet_bootstrap.cc:492] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=2} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:03:50.015596 7288 tablet_bootstrap.cc:492] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap complete.
I20240502 08:03:50.015731 7288 ts_tablet_manager.cc:1367] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.009s user 0.006s sys 0.000s
I20240502 08:03:50.015730 7298 tablet_copy_client.cc:523] T 68f96ca3354645fd91f666109eeaf847 P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:03:50.016049 7288 raft_consensus.cc:348] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 08:03:50.016151 7288 raft_consensus.cc:729] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: LEARNER
I20240502 08:03:50.016247 7288 consensus_queue.cc:260] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 08:03:50.016525 7288 ts_tablet_manager.cc:1398] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:03:50.017005 22606 tablet_copy_service.cc:342] P 8bea0fe8d8044a2685f92f7a10c9cf85: Request end of tablet copy session 725969d490a04fdfb3ba890c3f5096e3-a5374313bbba4b10904e0ea18851c07d received from {username='slave'} at 127.16.85.130:39611
I20240502 08:03:50.017071 22606 tablet_copy_service.cc:433] P 8bea0fe8d8044a2685f92f7a10c9cf85: ending tablet copy session 725969d490a04fdfb3ba890c3f5096e3-a5374313bbba4b10904e0ea18851c07d on tablet a5374313bbba4b10904e0ea18851c07d with peer 725969d490a04fdfb3ba890c3f5096e3
I20240502 08:03:50.018391 7298 tablet_bootstrap.cc:492] T 68f96ca3354645fd91f666109eeaf847 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 08:03:50.025548 7298 tablet_bootstrap.cc:492] T 68f96ca3354645fd91f666109eeaf847 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=2} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:03:50.025791 7298 tablet_bootstrap.cc:492] T 68f96ca3354645fd91f666109eeaf847 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap complete.
I20240502 08:03:50.025966 7298 ts_tablet_manager.cc:1367] T 68f96ca3354645fd91f666109eeaf847 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.008s user 0.005s sys 0.001s
I20240502 08:03:50.026335 7298 raft_consensus.cc:348] T 68f96ca3354645fd91f666109eeaf847 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 08:03:50.026463 7298 raft_consensus.cc:729] T 68f96ca3354645fd91f666109eeaf847 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: LEARNER
I20240502 08:03:50.026572 7298 consensus_queue.cc:260] T 68f96ca3354645fd91f666109eeaf847 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 1.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: VOTER last_known_addr { host: "127.16.85.129" port: 37451 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 08:03:50.026870 7298 ts_tablet_manager.cc:1398] T 68f96ca3354645fd91f666109eeaf847 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:03:50.027267 22737 tablet_copy_service.cc:342] P 725969d490a04fdfb3ba890c3f5096e3: Request end of tablet copy session 02a92e1c084b4308b9e855a4cc2c295f-68f96ca3354645fd91f666109eeaf847 received from {username='slave'} at 127.16.85.132:44073
I20240502 08:03:50.027334 22737 tablet_copy_service.cc:433] P 725969d490a04fdfb3ba890c3f5096e3: ending tablet copy session 02a92e1c084b4308b9e855a4cc2c295f-68f96ca3354645fd91f666109eeaf847 on tablet 68f96ca3354645fd91f666109eeaf847 with peer 02a92e1c084b4308b9e855a4cc2c295f
I20240502 08:03:50.036989 7288 ts_tablet_manager.cc:902] T a071d3a1489e4d97ae6ceea4b0478459 P 725969d490a04fdfb3ba890c3f5096e3: Initiating tablet copy from peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 08:03:50.037377 7288 tablet_copy_client.cc:308] T a071d3a1489e4d97ae6ceea4b0478459 P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.133:35459
I20240502 08:03:50.037722 24892 tablet_copy_service.cc:140] P 6d02017318a2461796d431962fe6a537: Received BeginTabletCopySession request for tablet a071d3a1489e4d97ae6ceea4b0478459 from peer 725969d490a04fdfb3ba890c3f5096e3 ({username='slave'} at 127.16.85.130:51383)
I20240502 08:03:50.037819 24892 tablet_copy_service.cc:161] P 6d02017318a2461796d431962fe6a537: Beginning new tablet copy session on tablet a071d3a1489e4d97ae6ceea4b0478459 from peer 725969d490a04fdfb3ba890c3f5096e3 at {username='slave'} at 127.16.85.130:51383: session id = 725969d490a04fdfb3ba890c3f5096e3-a071d3a1489e4d97ae6ceea4b0478459
I20240502 08:03:50.038581 24892 tablet_copy_source_session.cc:205] T a071d3a1489e4d97ae6ceea4b0478459 P 6d02017318a2461796d431962fe6a537: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:03:50.039242 7288 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a071d3a1489e4d97ae6ceea4b0478459. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:03:50.040939 7288 tablet_copy_client.cc:783] T a071d3a1489e4d97ae6ceea4b0478459 P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Starting download of 0 data blocks...
I20240502 08:03:50.041065 7288 tablet_copy_client.cc:647] T a071d3a1489e4d97ae6ceea4b0478459 P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Starting download of 1 WAL segments...
I20240502 08:03:50.042148 7288 tablet_copy_client.cc:523] T a071d3a1489e4d97ae6ceea4b0478459 P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:03:50.043300 7288 tablet_bootstrap.cc:492] T a071d3a1489e4d97ae6ceea4b0478459 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 08:03:50.047426 7288 tablet_bootstrap.cc:492] T a071d3a1489e4d97ae6ceea4b0478459 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:03:50.047649 7288 tablet_bootstrap.cc:492] T a071d3a1489e4d97ae6ceea4b0478459 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap complete.
I20240502 08:03:50.047801 7288 ts_tablet_manager.cc:1367] T a071d3a1489e4d97ae6ceea4b0478459 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240502 08:03:50.048178 7288 raft_consensus.cc:348] T a071d3a1489e4d97ae6ceea4b0478459 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 08:03:50.048290 7288 raft_consensus.cc:729] T a071d3a1489e4d97ae6ceea4b0478459 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: LEARNER
I20240502 08:03:50.048413 7288 consensus_queue.cc:260] T a071d3a1489e4d97ae6ceea4b0478459 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: false } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 08:03:50.049082 7288 ts_tablet_manager.cc:1398] T a071d3a1489e4d97ae6ceea4b0478459 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 08:03:50.049487 24892 tablet_copy_service.cc:342] P 6d02017318a2461796d431962fe6a537: Request end of tablet copy session 725969d490a04fdfb3ba890c3f5096e3-a071d3a1489e4d97ae6ceea4b0478459 received from {username='slave'} at 127.16.85.130:51383
I20240502 08:03:50.049552 24892 tablet_copy_service.cc:433] P 6d02017318a2461796d431962fe6a537: ending tablet copy session 725969d490a04fdfb3ba890c3f5096e3-a071d3a1489e4d97ae6ceea4b0478459 on tablet a071d3a1489e4d97ae6ceea4b0478459 with peer 725969d490a04fdfb3ba890c3f5096e3
I20240502 08:03:50.051103 7336 ts_tablet_manager.cc:902] T c3a7b2e52ba641e993cf1d5dc9724473 P 7db4178eed174225bfb24f5f9445ce88: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 08:03:50.051602 7336 tablet_copy_client.cc:308] T c3a7b2e52ba641e993cf1d5dc9724473 P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33387
I20240502 08:03:50.053099 22737 tablet_copy_service.cc:140] P 725969d490a04fdfb3ba890c3f5096e3: Received BeginTabletCopySession request for tablet c3a7b2e52ba641e993cf1d5dc9724473 from peer 7db4178eed174225bfb24f5f9445ce88 ({username='slave'} at 127.16.85.131:58387)
I20240502 08:03:50.053197 22737 tablet_copy_service.cc:161] P 725969d490a04fdfb3ba890c3f5096e3: Beginning new tablet copy session on tablet c3a7b2e52ba641e993cf1d5dc9724473 from peer 7db4178eed174225bfb24f5f9445ce88 at {username='slave'} at 127.16.85.131:58387: session id = 7db4178eed174225bfb24f5f9445ce88-c3a7b2e52ba641e993cf1d5dc9724473
I20240502 08:03:50.054057 22737 tablet_copy_source_session.cc:205] T c3a7b2e52ba641e993cf1d5dc9724473 P 725969d490a04fdfb3ba890c3f5096e3: Tablet Copy: opened 0 blocks and 1 log segments
W20240502 08:03:50.054749 24807 consensus_peers.cc:485] T 90acd69ebaba4fb2a67e16172e8d7aa5 P 6d02017318a2461796d431962fe6a537 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
I20240502 08:03:50.055616 7336 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c3a7b2e52ba641e993cf1d5dc9724473. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:03:50.057878 7336 tablet_copy_client.cc:783] T c3a7b2e52ba641e993cf1d5dc9724473 P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Starting download of 0 data blocks...
I20240502 08:03:50.058013 7336 tablet_copy_client.cc:647] T c3a7b2e52ba641e993cf1d5dc9724473 P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Starting download of 1 WAL segments...
I20240502 08:03:50.059028 7336 tablet_copy_client.cc:523] T c3a7b2e52ba641e993cf1d5dc9724473 P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:03:50.060011 7336 tablet_bootstrap.cc:492] T c3a7b2e52ba641e993cf1d5dc9724473 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 08:03:50.063262 7336 tablet_bootstrap.cc:492] T c3a7b2e52ba641e993cf1d5dc9724473 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:03:50.063464 7336 tablet_bootstrap.cc:492] T c3a7b2e52ba641e993cf1d5dc9724473 P 7db4178eed174225bfb24f5f9445ce88: Bootstrap complete.
I20240502 08:03:50.063586 7336 ts_tablet_manager.cc:1367] T c3a7b2e52ba641e993cf1d5dc9724473 P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
I20240502 08:03:50.063861 7336 raft_consensus.cc:348] T c3a7b2e52ba641e993cf1d5dc9724473 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 08:03:50.063946 7336 raft_consensus.cc:729] T c3a7b2e52ba641e993cf1d5dc9724473 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: LEARNER
I20240502 08:03:50.064038 7336 consensus_queue.cc:260] T c3a7b2e52ba641e993cf1d5dc9724473 P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 08:03:50.064262 7336 ts_tablet_manager.cc:1398] T c3a7b2e52ba641e993cf1d5dc9724473 P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:03:50.064626 22737 tablet_copy_service.cc:342] P 725969d490a04fdfb3ba890c3f5096e3: Request end of tablet copy session 7db4178eed174225bfb24f5f9445ce88-c3a7b2e52ba641e993cf1d5dc9724473 received from {username='slave'} at 127.16.85.131:58387
I20240502 08:03:50.064720 22737 tablet_copy_service.cc:433] P 725969d490a04fdfb3ba890c3f5096e3: ending tablet copy session 7db4178eed174225bfb24f5f9445ce88-c3a7b2e52ba641e993cf1d5dc9724473 on tablet c3a7b2e52ba641e993cf1d5dc9724473 with peer 7db4178eed174225bfb24f5f9445ce88
I20240502 08:03:50.100782 7348 ts_tablet_manager.cc:902] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 8bea0fe8d8044a2685f92f7a10c9cf85: Initiating tablet copy from peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459)
I20240502 08:03:50.101214 7348 tablet_copy_client.cc:308] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.133:35459
I20240502 08:03:50.101583 24892 tablet_copy_service.cc:140] P 6d02017318a2461796d431962fe6a537: Received BeginTabletCopySession request for tablet 7eb1b4bcf48b4cbea5cfb4dd64de2c70 from peer 8bea0fe8d8044a2685f92f7a10c9cf85 ({username='slave'} at 127.16.85.129:44115)
I20240502 08:03:50.101667 24892 tablet_copy_service.cc:161] P 6d02017318a2461796d431962fe6a537: Beginning new tablet copy session on tablet 7eb1b4bcf48b4cbea5cfb4dd64de2c70 from peer 8bea0fe8d8044a2685f92f7a10c9cf85 at {username='slave'} at 127.16.85.129:44115: session id = 8bea0fe8d8044a2685f92f7a10c9cf85-7eb1b4bcf48b4cbea5cfb4dd64de2c70
I20240502 08:03:50.102505 24892 tablet_copy_source_session.cc:205] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 6d02017318a2461796d431962fe6a537: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:03:50.103219 7348 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7eb1b4bcf48b4cbea5cfb4dd64de2c70. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:03:50.104892 7348 tablet_copy_client.cc:783] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet copy: Starting download of 0 data blocks...
I20240502 08:03:50.105031 7348 tablet_copy_client.cc:647] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet copy: Starting download of 1 WAL segments...
I20240502 08:03:50.105947 7348 tablet_copy_client.cc:523] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:03:50.106849 7348 tablet_bootstrap.cc:492] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 08:03:50.109915 7348 tablet_bootstrap.cc:492] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:03:50.110145 7348 tablet_bootstrap.cc:492] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap complete.
I20240502 08:03:50.110291 7348 ts_tablet_manager.cc:1367] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.003s user 0.003s sys 0.000s
I20240502 08:03:50.110642 7348 raft_consensus.cc:348] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } }
I20240502 08:03:50.110749 7348 raft_consensus.cc:729] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: LEARNER
I20240502 08:03:50.110858 7348 consensus_queue.cc:260] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } }
I20240502 08:03:50.111155 7348 ts_tablet_manager.cc:1398] T 7eb1b4bcf48b4cbea5cfb4dd64de2c70 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:03:50.111559 24892 tablet_copy_service.cc:342] P 6d02017318a2461796d431962fe6a537: Request end of tablet copy session 8bea0fe8d8044a2685f92f7a10c9cf85-7eb1b4bcf48b4cbea5cfb4dd64de2c70 received from {username='slave'} at 127.16.85.129:44115
I20240502 08:03:50.111621 24892 tablet_copy_service.cc:433] P 6d02017318a2461796d431962fe6a537: ending tablet copy session 8bea0fe8d8044a2685f92f7a10c9cf85-7eb1b4bcf48b4cbea5cfb4dd64de2c70 on tablet 7eb1b4bcf48b4cbea5cfb4dd64de2c70 with peer 8bea0fe8d8044a2685f92f7a10c9cf85
I20240502 08:03:50.125411 7288 ts_tablet_manager.cc:902] T 4679a4f9194a4dbcb85c28e31f6ac894 P 725969d490a04fdfb3ba890c3f5096e3: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
I20240502 08:03:50.125821 7288 tablet_copy_client.cc:308] T 4679a4f9194a4dbcb85c28e31f6ac894 P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.132:39425
I20240502 08:03:50.126246 22999 tablet_copy_service.cc:140] P 02a92e1c084b4308b9e855a4cc2c295f: Received BeginTabletCopySession request for tablet 4679a4f9194a4dbcb85c28e31f6ac894 from peer 725969d490a04fdfb3ba890c3f5096e3 ({username='slave'} at 127.16.85.130:53121)
I20240502 08:03:50.126345 22999 tablet_copy_service.cc:161] P 02a92e1c084b4308b9e855a4cc2c295f: Beginning new tablet copy session on tablet 4679a4f9194a4dbcb85c28e31f6ac894 from peer 725969d490a04fdfb3ba890c3f5096e3 at {username='slave'} at 127.16.85.130:53121: session id = 725969d490a04fdfb3ba890c3f5096e3-4679a4f9194a4dbcb85c28e31f6ac894
I20240502 08:03:50.127085 22999 tablet_copy_source_session.cc:205] T 4679a4f9194a4dbcb85c28e31f6ac894 P 02a92e1c084b4308b9e855a4cc2c295f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:03:50.127724 7288 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4679a4f9194a4dbcb85c28e31f6ac894. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:03:50.129416 7288 tablet_copy_client.cc:783] T 4679a4f9194a4dbcb85c28e31f6ac894 P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Starting download of 0 data blocks...
I20240502 08:03:50.129563 7288 tablet_copy_client.cc:647] T 4679a4f9194a4dbcb85c28e31f6ac894 P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Starting download of 1 WAL segments...
I20240502 08:03:50.130491 7288 tablet_copy_client.cc:523] T 4679a4f9194a4dbcb85c28e31f6ac894 P 725969d490a04fdfb3ba890c3f5096e3: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:03:50.131337 7288 tablet_bootstrap.cc:492] T 4679a4f9194a4dbcb85c28e31f6ac894 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap starting.
I20240502 08:03:50.133857 7288 tablet_bootstrap.cc:492] T 4679a4f9194a4dbcb85c28e31f6ac894 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:03:50.134083 7288 tablet_bootstrap.cc:492] T 4679a4f9194a4dbcb85c28e31f6ac894 P 725969d490a04fdfb3ba890c3f5096e3: Bootstrap complete.
I20240502 08:03:50.134222 7288 ts_tablet_manager.cc:1367] T 4679a4f9194a4dbcb85c28e31f6ac894 P 725969d490a04fdfb3ba890c3f5096e3: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.001s
I20240502 08:03:50.134549 7288 raft_consensus.cc:348] T 4679a4f9194a4dbcb85c28e31f6ac894 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 08:03:50.134649 7288 raft_consensus.cc:729] T 4679a4f9194a4dbcb85c28e31f6ac894 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 725969d490a04fdfb3ba890c3f5096e3, State: Initialized, Role: LEARNER
I20240502 08:03:50.134739 7288 consensus_queue.cc:260] T 4679a4f9194a4dbcb85c28e31f6ac894 P 725969d490a04fdfb3ba890c3f5096e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: NON_VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { promote: true } }
I20240502 08:03:50.134972 7288 ts_tablet_manager.cc:1398] T 4679a4f9194a4dbcb85c28e31f6ac894 P 725969d490a04fdfb3ba890c3f5096e3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:03:50.135398 22999 tablet_copy_service.cc:342] P 02a92e1c084b4308b9e855a4cc2c295f: Request end of tablet copy session 725969d490a04fdfb3ba890c3f5096e3-4679a4f9194a4dbcb85c28e31f6ac894 received from {username='slave'} at 127.16.85.130:53121
I20240502 08:03:50.135481 22999 tablet_copy_service.cc:433] P 02a92e1c084b4308b9e855a4cc2c295f: ending tablet copy session 725969d490a04fdfb3ba890c3f5096e3-4679a4f9194a4dbcb85c28e31f6ac894 on tablet 4679a4f9194a4dbcb85c28e31f6ac894 with peer 725969d490a04fdfb3ba890c3f5096e3
I20240502 08:03:50.141053 7104 consensus_queue.cc:579] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Leader has been unable to successfully communicate with peer 74f9aa91c2c448f980b512fe2e8d672e for more than 300 seconds (300.539s)
I20240502 08:03:50.144232 22847 consensus_queue.cc:237] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } }
I20240502 08:03:50.145516 22714 raft_consensus.cc:1261] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Refusing update from remote peer 7db4178eed174225bfb24f5f9445ce88: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 08:03:50.145757 22784 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205): Couldn't send request to peer 4654231dfe994c1a9d2e37d9c3d75699. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6dbf36c0f044466ab13a53bfe0d6cb53. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:50.145849 6889 consensus_queue.cc:1035] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [LEADER]: Connected to new peer: Peer: permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240502 08:03:50.146004 22783 consensus_peers.cc:485] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:50.146433 7348 ts_tablet_manager.cc:902] T 07409f4ddd0040fc98e6b2064e2df523 P 8bea0fe8d8044a2685f92f7a10c9cf85: Initiating tablet copy from peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 08:03:50.146816 7348 tablet_copy_client.cc:308] T 07409f4ddd0040fc98e6b2064e2df523 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.131:44055
I20240502 08:03:50.147200 22868 tablet_copy_service.cc:140] P 7db4178eed174225bfb24f5f9445ce88: Received BeginTabletCopySession request for tablet 07409f4ddd0040fc98e6b2064e2df523 from peer 8bea0fe8d8044a2685f92f7a10c9cf85 ({username='slave'} at 127.16.85.129:51639)
I20240502 08:03:50.147286 22868 tablet_copy_service.cc:161] P 7db4178eed174225bfb24f5f9445ce88: Beginning new tablet copy session on tablet 07409f4ddd0040fc98e6b2064e2df523 from peer 8bea0fe8d8044a2685f92f7a10c9cf85 at {username='slave'} at 127.16.85.129:51639: session id = 8bea0fe8d8044a2685f92f7a10c9cf85-07409f4ddd0040fc98e6b2064e2df523
I20240502 08:03:50.147320 7104 raft_consensus.cc:2932] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 08:03:50.148150 22868 tablet_copy_source_session.cc:205] T 07409f4ddd0040fc98e6b2064e2df523 P 7db4178eed174225bfb24f5f9445ce88: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:03:50.148597 22431 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 6dbf36c0f044466ab13a53bfe0d6cb53 with cas_config_opid_index 6: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:03:50.148947 22714 raft_consensus.cc:2932] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } } }
I20240502 08:03:50.149325 7348 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 07409f4ddd0040fc98e6b2064e2df523. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:03:50.150576 7228 consensus_queue.cc:579] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Leader has been unable to successfully communicate with peer 74f9aa91c2c448f980b512fe2e8d672e for more than 300 seconds (300.082s)
I20240502 08:03:50.151404 7348 tablet_copy_client.cc:783] T 07409f4ddd0040fc98e6b2064e2df523 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet copy: Starting download of 0 data blocks...
I20240502 08:03:50.151546 7348 tablet_copy_client.cc:647] T 07409f4ddd0040fc98e6b2064e2df523 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet copy: Starting download of 1 WAL segments...
I20240502 08:03:50.153206 22438 catalog_manager.cc:5484] T 6dbf36c0f044466ab13a53bfe0d6cb53 P 7db4178eed174225bfb24f5f9445ce88 reported cstate change: config changed from index 6 to 7, NON_VOTER 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135) added. New cstate: current_term: 1 leader_uuid: "7db4178eed174225bfb24f5f9445ce88" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: NON_VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:03:50.153466 7348 tablet_copy_client.cc:523] T 07409f4ddd0040fc98e6b2064e2df523 P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:03:50.154157 22979 consensus_queue.cc:237] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } }
I20240502 08:03:50.155408 25134 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:03:50.155508 22847 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:03:50.155856 6074 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:03:50.155992 6483 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:03:50.157879 6483 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 08:03:50.158828 25133 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 08:03:50.159547 22429 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f1fa312fe8e44d23970c5e3ba98c2c58 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:03:50.161237 22847 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } }
I20240502 08:03:50.161505 7348 tablet_bootstrap.cc:492] T 07409f4ddd0040fc98e6b2064e2df523 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 08:03:50.161022 22435 catalog_manager.cc:5484] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 8 to 9, VOTER 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134) evicted. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:03:50.164899 7348 tablet_bootstrap.cc:492] T 07409f4ddd0040fc98e6b2064e2df523 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:03:50.165122 7348 tablet_bootstrap.cc:492] T 07409f4ddd0040fc98e6b2064e2df523 P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap complete.
I20240502 08:03:50.165345 7348 ts_tablet_manager.cc:1367] T 07409f4ddd0040fc98e6b2064e2df523 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.004s user 0.004s sys 0.000s
I20240502 08:03:50.165712 7348 raft_consensus.cc:348] T 07409f4ddd0040fc98e6b2064e2df523 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } }
I20240502 08:03:50.166014 7348 raft_consensus.cc:729] T 07409f4ddd0040fc98e6b2064e2df523 P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: LEARNER
I20240502 08:03:50.166119 7348 consensus_queue.cc:260] T 07409f4ddd0040fc98e6b2064e2df523 P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } }
I20240502 08:03:50.166575 7348 ts_tablet_manager.cc:1398] T 07409f4ddd0040fc98e6b2064e2df523 P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 08:03:50.167033 22868 tablet_copy_service.cc:342] P 7db4178eed174225bfb24f5f9445ce88: Request end of tablet copy session 8bea0fe8d8044a2685f92f7a10c9cf85-07409f4ddd0040fc98e6b2064e2df523 received from {username='slave'} at 127.16.85.129:51639
I20240502 08:03:50.167086 22868 tablet_copy_service.cc:433] P 7db4178eed174225bfb24f5f9445ce88: ending tablet copy session 8bea0fe8d8044a2685f92f7a10c9cf85-07409f4ddd0040fc98e6b2064e2df523 on tablet 07409f4ddd0040fc98e6b2064e2df523 with peer 8bea0fe8d8044a2685f92f7a10c9cf85
I20240502 08:03:50.167024 22979 consensus_queue.cc:237] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
W20240502 08:03:50.168205 22912 consensus_peers.cc:485] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f -> Peer 6d02017318a2461796d431962fe6a537 (127.16.85.133:35459): Couldn't send request to peer 6d02017318a2461796d431962fe6a537. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f1fa312fe8e44d23970c5e3ba98c2c58. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:03:50.168269 22847 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:03:50.168563 25133 raft_consensus.cc:1261] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Refusing update from remote peer 02a92e1c084b4308b9e855a4cc2c295f: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:03:50.168848 6074 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:03:50.168987 7228 consensus_queue.cc:1035] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:03:50.171367 7228 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 08:03:50.173099 22429 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet f1fa312fe8e44d23970c5e3ba98c2c58 with cas_config_opid_index 9: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:03:50.174423 22436 catalog_manager.cc:5484] T f1fa312fe8e44d23970c5e3ba98c2c58 P 02a92e1c084b4308b9e855a4cc2c295f reported cstate change: config changed from index 9 to 10, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New cstate: current_term: 1 leader_uuid: "02a92e1c084b4308b9e855a4cc2c295f" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:03:50.174935 25133 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 4654231dfe994c1a9d2e37d9c3d75699 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 08:03:50.177584 22847 raft_consensus.cc:2932] T f1fa312fe8e44d23970c5e3ba98c2c58 P 7db4178eed174225bfb24f5f9445ce88 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, NON_VOTER 6d02017318a2461796d431962fe6a537 (127.16.85.133) added. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } } }
I20240502 08:03:50.217085 22764 maintenance_manager.cc:419] P 725969d490a04fdfb3ba890c3f5096e3: Scheduling UndoDeltaBlockGCOp(a5374313bbba4b10904e0ea18851c07d): 4840 bytes on disk
I20240502 08:03:50.217470 22654 maintenance_manager.cc:643] P 725969d490a04fdfb3ba890c3f5096e3: UndoDeltaBlockGCOp(a5374313bbba4b10904e0ea18851c07d) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":42,"lbm_reads_lt_1ms":4}
I20240502 08:03:50.231017 7348 ts_tablet_manager.cc:902] T bd1c2abfe2a3416bb10d6c14d3fed46a P 8bea0fe8d8044a2685f92f7a10c9cf85: Initiating tablet copy from peer 7db4178eed174225bfb24f5f9445ce88 (127.16.85.131:44055)
I20240502 08:03:50.231456 7348 tablet_copy_client.cc:308] T bd1c2abfe2a3416bb10d6c14d3fed46a P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.131:44055
I20240502 08:03:50.231850 22868 tablet_copy_service.cc:140] P 7db4178eed174225bfb24f5f9445ce88: Received BeginTabletCopySession request for tablet bd1c2abfe2a3416bb10d6c14d3fed46a from peer 8bea0fe8d8044a2685f92f7a10c9cf85 ({username='slave'} at 127.16.85.129:51639)
I20240502 08:03:50.231935 22868 tablet_copy_service.cc:161] P 7db4178eed174225bfb24f5f9445ce88: Beginning new tablet copy session on tablet bd1c2abfe2a3416bb10d6c14d3fed46a from peer 8bea0fe8d8044a2685f92f7a10c9cf85 at {username='slave'} at 127.16.85.129:51639: session id = 8bea0fe8d8044a2685f92f7a10c9cf85-bd1c2abfe2a3416bb10d6c14d3fed46a
I20240502 08:03:50.232995 22868 tablet_copy_source_session.cc:205] T bd1c2abfe2a3416bb10d6c14d3fed46a P 7db4178eed174225bfb24f5f9445ce88: Tablet Copy: opened 5 blocks and 1 log segments
I20240502 08:03:50.233891 7348 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bd1c2abfe2a3416bb10d6c14d3fed46a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:03:50.235829 7348 tablet_copy_client.cc:783] T bd1c2abfe2a3416bb10d6c14d3fed46a P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet copy: Starting download of 5 data blocks...
I20240502 08:03:50.240794 7348 tablet_copy_client.cc:647] T bd1c2abfe2a3416bb10d6c14d3fed46a P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet copy: Starting download of 1 WAL segments...
I20240502 08:03:50.242147 7298 ts_tablet_manager.cc:902] T 2b5aafb8ddb24e51867f856884a6ee33 P 02a92e1c084b4308b9e855a4cc2c295f: Initiating tablet copy from peer 7eced220cf1642f3873fc385b6191e69 (127.16.85.136:34769)
I20240502 08:03:50.242542 7298 tablet_copy_client.cc:308] T 2b5aafb8ddb24e51867f856884a6ee33 P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.136:34769
I20240502 08:03:50.242801 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling UndoDeltaBlockGCOp(68f96ca3354645fd91f666109eeaf847): 4925 bytes on disk
I20240502 08:03:50.242882 25280 tablet_copy_service.cc:140] P 7eced220cf1642f3873fc385b6191e69: Received BeginTabletCopySession request for tablet 2b5aafb8ddb24e51867f856884a6ee33 from peer 02a92e1c084b4308b9e855a4cc2c295f ({username='slave'} at 127.16.85.132:58659)
I20240502 08:03:50.242964 25280 tablet_copy_service.cc:161] P 7eced220cf1642f3873fc385b6191e69: Beginning new tablet copy session on tablet 2b5aafb8ddb24e51867f856884a6ee33 from peer 02a92e1c084b4308b9e855a4cc2c295f at {username='slave'} at 127.16.85.132:58659: session id = 02a92e1c084b4308b9e855a4cc2c295f-2b5aafb8ddb24e51867f856884a6ee33
I20240502 08:03:50.243153 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: UndoDeltaBlockGCOp(68f96ca3354645fd91f666109eeaf847) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":50,"lbm_reads_lt_1ms":4}
I20240502 08:03:50.243858 25280 tablet_copy_source_session.cc:205] T 2b5aafb8ddb24e51867f856884a6ee33 P 7eced220cf1642f3873fc385b6191e69: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:03:50.244583 7298 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2b5aafb8ddb24e51867f856884a6ee33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:03:50.246693 7298 tablet_copy_client.cc:783] T 2b5aafb8ddb24e51867f856884a6ee33 P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Starting download of 0 data blocks...
I20240502 08:03:50.246809 7298 tablet_copy_client.cc:647] T 2b5aafb8ddb24e51867f856884a6ee33 P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Starting download of 1 WAL segments...
I20240502 08:03:50.247934 7298 tablet_copy_client.cc:523] T 2b5aafb8ddb24e51867f856884a6ee33 P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:03:50.248625 7348 tablet_copy_client.cc:523] T bd1c2abfe2a3416bb10d6c14d3fed46a P 8bea0fe8d8044a2685f92f7a10c9cf85: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:03:50.248833 7298 tablet_bootstrap.cc:492] T 2b5aafb8ddb24e51867f856884a6ee33 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 08:03:50.250166 7348 tablet_bootstrap.cc:492] T bd1c2abfe2a3416bb10d6c14d3fed46a P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap starting.
I20240502 08:03:50.251912 7298 tablet_bootstrap.cc:492] T 2b5aafb8ddb24e51867f856884a6ee33 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:03:50.252107 7298 tablet_bootstrap.cc:492] T 2b5aafb8ddb24e51867f856884a6ee33 P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap complete.
I20240502 08:03:50.252244 7298 ts_tablet_manager.cc:1367] T 2b5aafb8ddb24e51867f856884a6ee33 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:03:50.252568 7298 raft_consensus.cc:348] T 2b5aafb8ddb24e51867f856884a6ee33 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 08:03:50.252691 7298 raft_consensus.cc:729] T 2b5aafb8ddb24e51867f856884a6ee33 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: LEARNER
I20240502 08:03:50.252782 7298 consensus_queue.cc:260] T 2b5aafb8ddb24e51867f856884a6ee33 P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 08:03:50.253039 7298 ts_tablet_manager.cc:1398] T 2b5aafb8ddb24e51867f856884a6ee33 P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 08:03:50.253720 25280 tablet_copy_service.cc:342] P 7eced220cf1642f3873fc385b6191e69: Request end of tablet copy session 02a92e1c084b4308b9e855a4cc2c295f-2b5aafb8ddb24e51867f856884a6ee33 received from {username='slave'} at 127.16.85.132:58659
I20240502 08:03:50.253780 25280 tablet_copy_service.cc:433] P 7eced220cf1642f3873fc385b6191e69: ending tablet copy session 02a92e1c084b4308b9e855a4cc2c295f-2b5aafb8ddb24e51867f856884a6ee33 on tablet 2b5aafb8ddb24e51867f856884a6ee33 with peer 02a92e1c084b4308b9e855a4cc2c295f
I20240502 08:03:50.259021 7348 tablet_bootstrap.cc:492] T bd1c2abfe2a3416bb10d6c14d3fed46a P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=2} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:03:50.259286 7348 tablet_bootstrap.cc:492] T bd1c2abfe2a3416bb10d6c14d3fed46a P 8bea0fe8d8044a2685f92f7a10c9cf85: Bootstrap complete.
I20240502 08:03:50.259465 7348 ts_tablet_manager.cc:1367] T bd1c2abfe2a3416bb10d6c14d3fed46a P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent bootstrapping tablet: real 0.009s user 0.001s sys 0.005s
I20240502 08:03:50.259868 7348 raft_consensus.cc:348] T bd1c2abfe2a3416bb10d6c14d3fed46a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } }
I20240502 08:03:50.259958 7348 raft_consensus.cc:729] T bd1c2abfe2a3416bb10d6c14d3fed46a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8bea0fe8d8044a2685f92f7a10c9cf85, State: Initialized, Role: LEARNER
I20240502 08:03:50.260025 7348 consensus_queue.cc:260] T bd1c2abfe2a3416bb10d6c14d3fed46a P 8bea0fe8d8044a2685f92f7a10c9cf85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 1.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "8bea0fe8d8044a2685f92f7a10c9cf85" member_type: NON_VOTER last_known_addr { host: "127.16.85.129" port: 37451 } attrs { promote: true } }
I20240502 08:03:50.260260 7348 ts_tablet_manager.cc:1398] T bd1c2abfe2a3416bb10d6c14d3fed46a P 8bea0fe8d8044a2685f92f7a10c9cf85: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:03:50.260710 22868 tablet_copy_service.cc:342] P 7db4178eed174225bfb24f5f9445ce88: Request end of tablet copy session 8bea0fe8d8044a2685f92f7a10c9cf85-bd1c2abfe2a3416bb10d6c14d3fed46a received from {username='slave'} at 127.16.85.129:51639
I20240502 08:03:50.260774 22868 tablet_copy_service.cc:433] P 7db4178eed174225bfb24f5f9445ce88: ending tablet copy session 8bea0fe8d8044a2685f92f7a10c9cf85-bd1c2abfe2a3416bb10d6c14d3fed46a on tablet bd1c2abfe2a3416bb10d6c14d3fed46a with peer 8bea0fe8d8044a2685f92f7a10c9cf85
I20240502 08:03:50.268707 22633 maintenance_manager.cc:419] P 8bea0fe8d8044a2685f92f7a10c9cf85: Scheduling UndoDeltaBlockGCOp(bd1c2abfe2a3416bb10d6c14d3fed46a): 4925 bytes on disk
I20240502 08:03:50.269083 22523 maintenance_manager.cc:643] P 8bea0fe8d8044a2685f92f7a10c9cf85: UndoDeltaBlockGCOp(bd1c2abfe2a3416bb10d6c14d3fed46a) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":51,"lbm_reads_lt_1ms":4}
I20240502 08:03:50.323590 7298 ts_tablet_manager.cc:902] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 02a92e1c084b4308b9e855a4cc2c295f: Initiating tablet copy from peer 4654231dfe994c1a9d2e37d9c3d75699 (127.16.85.135:41205)
I20240502 08:03:50.324031 7298 tablet_copy_client.cc:308] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.135:41205
I20240502 08:03:50.324429 25155 tablet_copy_service.cc:140] P 4654231dfe994c1a9d2e37d9c3d75699: Received BeginTabletCopySession request for tablet d2eebf9aaa2c4f92a4fefbe5bb6feb0c from peer 02a92e1c084b4308b9e855a4cc2c295f ({username='slave'} at 127.16.85.132:50937)
I20240502 08:03:50.324517 25155 tablet_copy_service.cc:161] P 4654231dfe994c1a9d2e37d9c3d75699: Beginning new tablet copy session on tablet d2eebf9aaa2c4f92a4fefbe5bb6feb0c from peer 02a92e1c084b4308b9e855a4cc2c295f at {username='slave'} at 127.16.85.132:50937: session id = 02a92e1c084b4308b9e855a4cc2c295f-d2eebf9aaa2c4f92a4fefbe5bb6feb0c
I20240502 08:03:50.325302 25155 tablet_copy_source_session.cc:205] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 4654231dfe994c1a9d2e37d9c3d75699: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:03:50.325989 7298 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d2eebf9aaa2c4f92a4fefbe5bb6feb0c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:03:50.327718 7298 tablet_copy_client.cc:783] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Starting download of 0 data blocks...
I20240502 08:03:50.327853 7298 tablet_copy_client.cc:647] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Starting download of 1 WAL segments...
I20240502 08:03:50.328773 7298 tablet_copy_client.cc:523] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:03:50.329779 7298 tablet_bootstrap.cc:492] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 08:03:50.332659 7298 tablet_bootstrap.cc:492] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:03:50.332908 7298 tablet_bootstrap.cc:492] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap complete.
I20240502 08:03:50.333038 7298 ts_tablet_manager.cc:1367] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.002s
I20240502 08:03:50.333343 7298 raft_consensus.cc:348] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 08:03:50.333436 7298 raft_consensus.cc:729] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: LEARNER
I20240502 08:03:50.333524 7298 consensus_queue.cc:260] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } } peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: VOTER last_known_addr { host: "127.16.85.133" port: 35459 } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 08:03:50.333775 7298 ts_tablet_manager.cc:1398] T d2eebf9aaa2c4f92a4fefbe5bb6feb0c P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:03:50.334213 25155 tablet_copy_service.cc:342] P 4654231dfe994c1a9d2e37d9c3d75699: Request end of tablet copy session 02a92e1c084b4308b9e855a4cc2c295f-d2eebf9aaa2c4f92a4fefbe5bb6feb0c received from {username='slave'} at 127.16.85.132:50937
I20240502 08:03:50.334278 25155 tablet_copy_service.cc:433] P 4654231dfe994c1a9d2e37d9c3d75699: ending tablet copy session 02a92e1c084b4308b9e855a4cc2c295f-d2eebf9aaa2c4f92a4fefbe5bb6feb0c on tablet d2eebf9aaa2c4f92a4fefbe5bb6feb0c with peer 02a92e1c084b4308b9e855a4cc2c295f
I20240502 08:03:50.342259 7336 ts_tablet_manager.cc:902] T da39029564d042f6acfc23f99fb63dfd P 7db4178eed174225bfb24f5f9445ce88: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 08:03:50.342656 7336 tablet_copy_client.cc:308] T da39029564d042f6acfc23f99fb63dfd P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33387
I20240502 08:03:50.342988 22737 tablet_copy_service.cc:140] P 725969d490a04fdfb3ba890c3f5096e3: Received BeginTabletCopySession request for tablet da39029564d042f6acfc23f99fb63dfd from peer 7db4178eed174225bfb24f5f9445ce88 ({username='slave'} at 127.16.85.131:58387)
I20240502 08:03:50.343070 22737 tablet_copy_service.cc:161] P 725969d490a04fdfb3ba890c3f5096e3: Beginning new tablet copy session on tablet da39029564d042f6acfc23f99fb63dfd from peer 7db4178eed174225bfb24f5f9445ce88 at {username='slave'} at 127.16.85.131:58387: session id = 7db4178eed174225bfb24f5f9445ce88-da39029564d042f6acfc23f99fb63dfd
I20240502 08:03:50.344015 22737 tablet_copy_source_session.cc:205] T da39029564d042f6acfc23f99fb63dfd P 725969d490a04fdfb3ba890c3f5096e3: Tablet Copy: opened 0 blocks and 1 log segments
W20240502 08:03:50.344152 22783 consensus_peers.cc:485] T 28832d9e18ea41b194cf33d8c7971e23 P 7db4178eed174225bfb24f5f9445ce88 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
I20240502 08:03:50.344646 7336 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet da39029564d042f6acfc23f99fb63dfd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:03:50.346287 7336 tablet_copy_client.cc:783] T da39029564d042f6acfc23f99fb63dfd P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Starting download of 0 data blocks...
I20240502 08:03:50.346419 7336 tablet_copy_client.cc:647] T da39029564d042f6acfc23f99fb63dfd P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Starting download of 1 WAL segments...
I20240502 08:03:50.347291 7336 tablet_copy_client.cc:523] T da39029564d042f6acfc23f99fb63dfd P 7db4178eed174225bfb24f5f9445ce88: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:03:50.348268 7336 tablet_bootstrap.cc:492] T da39029564d042f6acfc23f99fb63dfd P 7db4178eed174225bfb24f5f9445ce88: Bootstrap starting.
I20240502 08:03:50.351861 7336 tablet_bootstrap.cc:492] T da39029564d042f6acfc23f99fb63dfd P 7db4178eed174225bfb24f5f9445ce88: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:03:50.352079 7336 tablet_bootstrap.cc:492] T da39029564d042f6acfc23f99fb63dfd P 7db4178eed174225bfb24f5f9445ce88: Bootstrap complete.
I20240502 08:03:50.352205 7336 ts_tablet_manager.cc:1367] T da39029564d042f6acfc23f99fb63dfd P 7db4178eed174225bfb24f5f9445ce88: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:03:50.352481 7336 raft_consensus.cc:348] T da39029564d042f6acfc23f99fb63dfd P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 08:03:50.352564 7336 raft_consensus.cc:729] T da39029564d042f6acfc23f99fb63dfd P 7db4178eed174225bfb24f5f9445ce88 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7db4178eed174225bfb24f5f9445ce88, State: Initialized, Role: LEARNER
I20240502 08:03:50.352676 7336 consensus_queue.cc:260] T da39029564d042f6acfc23f99fb63dfd P 7db4178eed174225bfb24f5f9445ce88 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4654231dfe994c1a9d2e37d9c3d75699" member_type: VOTER last_known_addr { host: "127.16.85.135" port: 41205 } attrs { replace: true } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: VOTER last_known_addr { host: "127.16.85.132" port: 39425 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: NON_VOTER last_known_addr { host: "127.16.85.131" port: 44055 } attrs { promote: true } }
I20240502 08:03:50.352967 7336 ts_tablet_manager.cc:1398] T da39029564d042f6acfc23f99fb63dfd P 7db4178eed174225bfb24f5f9445ce88: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 08:03:50.353374 22737 tablet_copy_service.cc:342] P 725969d490a04fdfb3ba890c3f5096e3: Request end of tablet copy session 7db4178eed174225bfb24f5f9445ce88-da39029564d042f6acfc23f99fb63dfd received from {username='slave'} at 127.16.85.131:58387
I20240502 08:03:50.353442 22737 tablet_copy_service.cc:433] P 725969d490a04fdfb3ba890c3f5096e3: ending tablet copy session 7db4178eed174225bfb24f5f9445ce88-da39029564d042f6acfc23f99fb63dfd on tablet da39029564d042f6acfc23f99fb63dfd with peer 7db4178eed174225bfb24f5f9445ce88
I20240502 08:03:50.420369 7298 ts_tablet_manager.cc:902] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 08:03:50.420818 7298 tablet_copy_client.cc:308] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33387
I20240502 08:03:50.421144 22737 tablet_copy_service.cc:140] P 725969d490a04fdfb3ba890c3f5096e3: Received BeginTabletCopySession request for tablet cceaa8887081401dbc9154d265004c7b from peer 02a92e1c084b4308b9e855a4cc2c295f ({username='slave'} at 127.16.85.132:44073)
I20240502 08:03:50.421223 22737 tablet_copy_service.cc:161] P 725969d490a04fdfb3ba890c3f5096e3: Beginning new tablet copy session on tablet cceaa8887081401dbc9154d265004c7b from peer 02a92e1c084b4308b9e855a4cc2c295f at {username='slave'} at 127.16.85.132:44073: session id = 02a92e1c084b4308b9e855a4cc2c295f-cceaa8887081401dbc9154d265004c7b
I20240502 08:03:50.422034 22737 tablet_copy_source_session.cc:205] T cceaa8887081401dbc9154d265004c7b P 725969d490a04fdfb3ba890c3f5096e3: Tablet Copy: opened 5 blocks and 1 log segments
I20240502 08:03:50.422652 7298 ts_tablet_manager.cc:1875] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 08:03:50.424464 7298 ts_tablet_manager.cc:1888] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.5
I20240502 08:03:50.424629 7298 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cceaa8887081401dbc9154d265004c7b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:03:50.425411 7298 tablet_copy_client.cc:783] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Starting download of 5 data blocks...
I20240502 08:03:50.429500 7298 tablet_copy_client.cc:647] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Starting download of 1 WAL segments...
I20240502 08:03:50.430660 7298 tablet_copy_client.cc:523] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:03:50.431614 7298 tablet_bootstrap.cc:492] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap starting.
I20240502 08:03:50.433529 7409 ts_tablet_manager.cc:902] T e25ba5cc77334a3595a4575fa98476f0 P 6d02017318a2461796d431962fe6a537: Initiating tablet copy from peer 725969d490a04fdfb3ba890c3f5096e3 (127.16.85.130:33387)
I20240502 08:03:50.433943 7409 tablet_copy_client.cc:308] T e25ba5cc77334a3595a4575fa98476f0 P 6d02017318a2461796d431962fe6a537: tablet copy: Beginning tablet copy session from remote peer at address 127.16.85.130:33387
I20240502 08:03:50.434423 22737 tablet_copy_service.cc:140] P 725969d490a04fdfb3ba890c3f5096e3: Received BeginTabletCopySession request for tablet e25ba5cc77334a3595a4575fa98476f0 from peer 6d02017318a2461796d431962fe6a537 ({username='slave'} at 127.16.85.133:53877)
I20240502 08:03:50.434509 22737 tablet_copy_service.cc:161] P 725969d490a04fdfb3ba890c3f5096e3: Beginning new tablet copy session on tablet e25ba5cc77334a3595a4575fa98476f0 from peer 6d02017318a2461796d431962fe6a537 at {username='slave'} at 127.16.85.133:53877: session id = 6d02017318a2461796d431962fe6a537-e25ba5cc77334a3595a4575fa98476f0
I20240502 08:03:50.435328 22737 tablet_copy_source_session.cc:205] T e25ba5cc77334a3595a4575fa98476f0 P 725969d490a04fdfb3ba890c3f5096e3: Tablet Copy: opened 5 blocks and 1 log segments
I20240502 08:03:50.436133 7409 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e25ba5cc77334a3595a4575fa98476f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:03:50.437709 7409 tablet_copy_client.cc:783] T e25ba5cc77334a3595a4575fa98476f0 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 5 data blocks...
I20240502 08:03:50.438320 7298 tablet_bootstrap.cc:492] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=2} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:03:50.438550 7298 tablet_bootstrap.cc:492] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: Bootstrap complete.
I20240502 08:03:50.438735 7298 ts_tablet_manager.cc:1367] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.006s
I20240502 08:03:50.439121 7298 raft_consensus.cc:348] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 08:03:50.439229 7298 raft_consensus.cc:729] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 02a92e1c084b4308b9e855a4cc2c295f, State: Initialized, Role: LEARNER
I20240502 08:03:50.439329 7298 consensus_queue.cc:260] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "7db4178eed174225bfb24f5f9445ce88" member_type: VOTER last_known_addr { host: "127.16.85.131" port: 44055 } } peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "02a92e1c084b4308b9e855a4cc2c295f" member_type: NON_VOTER last_known_addr { host: "127.16.85.132" port: 39425 } attrs { promote: true } }
I20240502 08:03:50.439580 7298 ts_tablet_manager.cc:1398] T cceaa8887081401dbc9154d265004c7b P 02a92e1c084b4308b9e855a4cc2c295f: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:03:50.440021 22736 tablet_copy_service.cc:342] P 725969d490a04fdfb3ba890c3f5096e3: Request end of tablet copy session 02a92e1c084b4308b9e855a4cc2c295f-cceaa8887081401dbc9154d265004c7b received from {username='slave'} at 127.16.85.132:44073
I20240502 08:03:50.440081 22736 tablet_copy_service.cc:433] P 725969d490a04fdfb3ba890c3f5096e3: ending tablet copy session 02a92e1c084b4308b9e855a4cc2c295f-cceaa8887081401dbc9154d265004c7b on tablet cceaa8887081401dbc9154d265004c7b with peer 02a92e1c084b4308b9e855a4cc2c295f
I20240502 08:03:50.441799 7409 tablet_copy_client.cc:647] T e25ba5cc77334a3595a4575fa98476f0 P 6d02017318a2461796d431962fe6a537: tablet copy: Starting download of 1 WAL segments...
I20240502 08:03:50.442831 7409 tablet_copy_client.cc:523] T e25ba5cc77334a3595a4575fa98476f0 P 6d02017318a2461796d431962fe6a537: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:03:50.444118 7409 tablet_bootstrap.cc:492] T e25ba5cc77334a3595a4575fa98476f0 P 6d02017318a2461796d431962fe6a537: Bootstrap starting.
I20240502 08:03:50.451249 7409 tablet_bootstrap.cc:492] T e25ba5cc77334a3595a4575fa98476f0 P 6d02017318a2461796d431962fe6a537: Bootstrap replayed 1/1 log segments. Stats: ops{read=10 overwritten=0 applied=10 ignored=2} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:03:50.451478 7409 tablet_bootstrap.cc:492] T e25ba5cc77334a3595a4575fa98476f0 P 6d02017318a2461796d431962fe6a537: Bootstrap complete.
I20240502 08:03:50.451635 7409 ts_tablet_manager.cc:1367] T e25ba5cc77334a3595a4575fa98476f0 P 6d02017318a2461796d431962fe6a537: Time spent bootstrapping tablet: real 0.008s user 0.004s sys 0.002s
I20240502 08:03:50.451956 7409 raft_consensus.cc:348] T e25ba5cc77334a3595a4575fa98476f0 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 08:03:50.452052 7409 raft_consensus.cc:729] T e25ba5cc77334a3595a4575fa98476f0 P 6d02017318a2461796d431962fe6a537 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6d02017318a2461796d431962fe6a537, State: Initialized, Role: LEARNER
I20240502 08:03:50.452139 7409 consensus_queue.cc:260] T e25ba5cc77334a3595a4575fa98476f0 P 6d02017318a2461796d431962fe6a537 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 10, Last appended: 1.10, Last appended by leader: 10, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "725969d490a04fdfb3ba890c3f5096e3" member_type: VOTER last_known_addr { host: "127.16.85.130" port: 33387 } } peers { permanent_uuid: "74f9aa91c2c448f980b512fe2e8d672e" member_type: VOTER last_known_addr { host: "127.16.85.134" port: 34857 } attrs { promote: false } } peers { permanent_uuid: "7eced220cf1642f3873fc385b6191e69" member_type: VOTER last_known_addr { host: "127.16.85.136" port: 34769 } attrs { promote: false } } peers { permanent_uuid: "6d02017318a2461796d431962fe6a537" member_type: NON_VOTER last_known_addr { host: "127.16.85.133" port: 35459 } attrs { promote: true } }
I20240502 08:03:50.452397 7409 ts_tablet_manager.cc:1398] T e25ba5cc77334a3595a4575fa98476f0 P 6d02017318a2461796d431962fe6a537: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:03:50.452839 22736 tablet_copy_service.cc:342] P 725969d490a04fdfb3ba890c3f5096e3: Request end of tablet copy session 6d02017318a2461796d431962fe6a537-e25ba5cc77334a3595a4575fa98476f0 received from {username='slave'} at 127.16.85.133:53877
I20240502 08:03:50.452909 22736 tablet_copy_service.cc:433] P 725969d490a04fdfb3ba890c3f5096e3: ending tablet copy session 6d02017318a2461796d431962fe6a537-e25ba5cc77334a3595a4575fa98476f0 on tablet e25ba5cc77334a3595a4575fa98476f0 with peer 6d02017318a2461796d431962fe6a537
I20240502 08:03:50.473268 24919 maintenance_manager.cc:419] P 6d02017318a2461796d431962fe6a537: Scheduling UndoDeltaBlockGCOp(e25ba5cc77334a3595a4575fa98476f0): 4925 bytes on disk
I20240502 08:03:50.473591 24809 maintenance_manager.cc:643] P 6d02017318a2461796d431962fe6a537: UndoDeltaBlockGCOp(e25ba5cc77334a3595a4575fa98476f0) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":37,"lbm_reads_lt_1ms":4}
I20240502 08:03:50.497045 23026 maintenance_manager.cc:419] P 02a92e1c084b4308b9e855a4cc2c295f: Scheduling UndoDeltaBlockGCOp(cceaa8887081401dbc9154d265004c7b): 4925 bytes on disk
I20240502 08:03:50.497396 22916 maintenance_manager.cc:643] P 02a92e1c084b4308b9e855a4cc2c295f: UndoDeltaBlockGCOp(cceaa8887081401dbc9154d265004c7b) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":47,"lbm_reads_lt_1ms":4}
I20240502 08:03:50.523125 22979 raft_consensus.cc:1203] T 68f96ca3354645fd91f666109eeaf847 P 02a92e1c084b4308b9e855a4cc2c295f [term 1 LEARNER]: Deduplicated request from leader. Original: 1.5->[1.6-1.6] Dedup: 1.6->[]
I20240502 08:03:50.523183 22714 raft_consensus.cc:1203] T a071d3a1489e4d97ae6ceea4b0478459 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
W20240502 08:03:50.541500 22430 catalog_manager.cc:4573] TS 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): DeleteTablet:TABLET_DATA_TOMBSTONED RPC failed for tablet f1fa312fe8e44d23970c5e3ba98c2c58: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111)
I20240502 08:03:50.600775 22716 raft_consensus.cc:1203] T a5374313bbba4b10904e0ea18851c07d P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:03:50.616350 22716 raft_consensus.cc:1203] T 4679a4f9194a4dbcb85c28e31f6ac894 P 725969d490a04fdfb3ba890c3f5096e3 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
W20240502 08:03:50.621932 25201 consensus_peers.cc:485] T cf0d3af4df9b410baaa088c744eeb2a6 P 7eced220cf1642f3873fc385b6191e69 -> Peer 74f9aa91c2c448f980b512fe2e8d672e (127.16.85.134:34857): Couldn't send request to peer 74f9aa91c2c448f980b512fe2e8d672e. Status: Network error: Client connection negotiation failed: client connection to 127.16.85.134:34857: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
I20240502 08:03:50.624085 22584 raft_consensus.cc:1203] T bd1c2abfe2a3416bb10d6c14d3fed46a P 8bea0fe8d8044a2685f92f7a10c9cf85 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.5->[1.6-1.6] Dedup: 1.6->[]
I20240502 08:03:50.632474 7409 ts_tablet_manager.cc:902] T f1fa312fe8e44d23970c5e3ba98c2c58 P 6d02017318a2461796d431962fe6a537: Initiating tablet copy from peer 02a92e1c084b4308b9e855a4cc2c295f (127.16.85.132:39425)
F20240502 08:03:50.632668 7409 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-023958ce03b84b2da063c13e4284e04d.queue_time_us, tablet-download-pool-023958ce03b84b2da063c13e4284e04d.queue_time_us) (tablet-download-pool-023958ce03b84b2da063c13e4284e04d.run_cpu_time_us, tablet-download-pool-023958ce03b84b2da063c13e4284e04d.run_cpu_time_us) (tablet-download-pool-023958ce03b84b2da063c13e4284e04d.run_wall_time_us, tablet-download-pool-023958ce03b84b2da063c13e4284e04d.run_wall_time_us) (tablet-download-pool-398405a3694c4d29b0627d2d74ebadc6.queue_time_us, tablet-download-pool-398405a3694c4d29b0627d2d74ebadc6.queue_time_us) (tablet-download-pool-398405a3694c4d29b0627d2d74ebadc6.run_cpu_time_us, tablet-download-pool-398405a3694c4d29b0627d2d74ebadc6.run_cpu_time_us) (tablet-download-pool-398405a3694c4d29b0627d2d74ebadc6.run_wall_time_us, tablet-download-pool-398405a3694c4d29b0627d2d74ebadc6.run_wall_time_us) (tablet-download-pool-4363e32404304568b94f615e4e29f883.queue_time_us, tablet-download-pool-4363e32404304568b94f615e4e29f883.queue_time_us) (tablet-download-pool-4363e32404304568b94f615e4e29f883.run_cpu_time_us, tablet-download-pool-4363e32404304568b94f615e4e29f883.run_cpu_time_us) (tablet-download-pool-4363e32404304568b94f615e4e29f883.run_wall_time_us, tablet-download-pool-4363e32404304568b94f615e4e29f883.run_wall_time_us) (tablet-download-pool-887c4c76980d4672aec02b2dbd4d8c89.queue_time_us, tablet-download-pool-887c4c76980d4672aec02b2dbd4d8c89.queue_time_us) (tablet-download-pool-887c4c76980d4672aec02b2dbd4d8c89.run_cpu_time_us, tablet-download-pool-887c4c76980d4672aec02b2dbd4d8c89.run_cpu_time_us) (tablet-download-pool-887c4c76980d4672aec02b2dbd4d8c89.run_wall_time_us, tablet-download-pool-887c4c76980d4672aec02b2dbd4d8c89.run_wall_time_us) (tablet-download-pool-9d2283e5d28347519b38e876d840608e.queue_time_us, tablet-download-pool-9d2283e5d28347519b38e876d840608e.queue_time_us) (tablet-download-pool-9d2283e5d28347519b38e876d840608e.run_cpu_time_us, tablet-download-pool-9d2283e5d28347519b38e876d840608e.run_cpu_time_us) (tablet-download-pool-9d2283e5d28347519b38e876d840608e.run_wall_time_us, tablet-download-pool-9d2283e5d28347519b38e876d840608e.run_wall_time_us) (tablet-download-pool-9e5fe86b44744df9b2e4d430fdec716f.queue_time_us, tablet-download-pool-9e5fe86b44744df9b2e4d430fdec716f.queue_time_us) (tablet-download-pool-9e5fe86b44744df9b2e4d430fdec716f.run_cpu_time_us, tablet-download-pool-9e5fe86b44744df9b2e4d430fdec716f.run_cpu_time_us) (tablet-download-pool-9e5fe86b44744df9b2e4d430fdec716f.run_wall_time_us, tablet-download-pool-9e5fe86b44744df9b2e4d430fdec716f.run_wall_time_us) (tablet-download-pool-b22f8e50a7d24dbdab665e47120602ce.queue_time_us, tablet-download-pool-b22f8e50a7d24dbdab665e47120602ce.queue_time_us) (tablet-download-pool-b22f8e50a7d24dbdab665e47120602ce.run_cpu_time_us, tablet-download-pool-b22f8e50a7d24dbdab665e47120602ce.run_cpu_time_us) (tablet-download-pool-b22f8e50a7d24dbdab665e47120602ce.run_wall_time_us, tablet-download-pool-b22f8e50a7d24dbdab665e47120602ce.run_wall_time_us) (tablet-download-pool-c53e8b3bd774466dbafe4994472975bc.queue_time_us, tablet-download-pool-c53e8b3bd774466dbafe4994472975bc.queue_time_us) (tablet-download-pool-c53e8b3bd774466dbafe4994472975bc.run_cpu_time_us, tablet-download-pool-c53e8b3bd774466dbafe4994472975bc.run_cpu_time_us) (tablet-download-pool-c53e8b3bd774466dbafe4994472975bc.run_wall_time_us, tablet-download-pool-c53e8b3bd774466dbafe4994472975bc.run_wall_time_us) (tablet-download-pool-ca85b40df34d4e1c91a55d7dd69e1494.queue_time_us, tablet-download-pool-ca85b40df34d4e1c91a55d7dd69e1494.queue_time_us) (tablet-download-pool-ca85b40df34d4e1c91a55d7dd69e1494.run_cpu_time_us, tablet-download-pool-ca85b40df34d4e1c91a55d7dd69e1494.run_cpu_time_us) (tablet-download-pool-ca85b40df34d4e1c91a55d7dd69e1494.run_wall_time_us, tablet-download-pool-ca85b40df34d4e1c91a55d7dd69e1494.run_wall_time_us) (tablet-download-pool-ddebb5e20e134841886be53f8f0d5e02.queue_time_us, tablet-download-pool-ddebb5e20e134841886be53f8f0d5e02.queue_time_us) (tablet-download-pool-ddebb5e20e134841886be53f8f0d5e02.run_cpu_time_us, tablet-download-pool-ddebb5e20e134841886be53f8f0d5e02.run_cpu_time_us) (tablet-download-pool-ddebb5e20e134841886be53f8f0d5e02.run_wall_time_us, tablet-download-pool-ddebb5e20e134841886be53f8f0d5e02.run_wall_time_us) (tablet-download-pool-e25ba5cc77334a3595a4575fa98476f0.queue_time_us, tablet-download-pool-e25ba5cc77334a3595a4575fa98476f0.queue_time_us) (tablet-download-pool-e25ba5cc77334a3595a4575fa98476f0.run_cpu_time_us, tablet-download-pool-e25ba5cc77334a3595a4575fa98476f0.run_cpu_time_us) (tablet-download-pool-e25ba5cc77334a3595a4575fa98476f0.run_wall_time_us, tablet-download-pool-e25ba5cc77334a3595a4575fa98476f0.run_wall_time_us) (tablet-download-pool-e5d5f67e026344998d5fb079f532d076.queue_time_us, tablet-download-pool-e5d5f67e026344998d5fb079f532d076.queue_time_us) (tablet-download-pool-e5d5f67e026344998d5fb079f532d076.run_cpu_time_us, tablet-download-pool-e5d5f67e026344998d5fb079f532d076.run_cpu_time_us) (tablet-download-pool-e5d5f67e026344998d5fb079f532d076.run_wall_time_us, tablet-download-pool-e5d5f67e026344998d5fb079f532d076.run_wall_time_us) (tablet-download-pool-ee06ff655f9c46c1bb06af2dd539fde7.queue_time_us, tablet-download-pool-ee06ff655f9c46c1bb06af2dd539fde7.queue_time_us) (tablet-download-pool-ee06ff655f9c46c1bb06af2dd539fde7.run_cpu_time_us, tablet-download-pool-ee06ff655f9c46c1bb06af2dd539fde7.run_cpu_time_us) (tablet-download-pool-ee06ff655f9c46c1bb06af2dd539fde7.run_wall_time_us, tablet-download-pool-ee06ff655f9c46c1bb06af2dd539fde7.run_wall_time_us) (tablet-download-pool-f1830cb65a294734ad80f274b89ac185.queue_time_us, tablet-download-pool-f1830cb65a294734ad80f274b89ac185.queue_time_us) (tablet-download-pool-f1830cb65a294734ad80f274b89ac185.run_cpu_time_us, tablet-download-pool-f1830cb65a294734ad80f274b89ac185.run_cpu_time_us) (tablet-download-pool-f1830cb65a294734ad80f274b89ac185.run_wall_time_us, tablet-download-pool-f1830cb65a294734ad80f274b89ac185.run_wall_time_us) (tablet-download-pool-f1fa312fe8e44d23970c5e3ba98c2c58.queue_time_us, tablet-download-pool-f1fa312fe8e44d23970c5e3ba98c2c58.queue_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registr